diff --git a/.editorconfig b/.editorconfig index cb3312517..443c4c284 100644 --- a/.editorconfig +++ b/.editorconfig @@ -3,6 +3,7 @@ root = true [src/*] end_of_line = lf insert_final_newline = true +max_line_length = 100 [ci/*.sh] indent_style = space diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a21e342cb..bb0493bf7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,10 +16,11 @@ jobs: env: MDBOOK_VERSION: 0.4.21 MDBOOK_LINKCHECK_VERSION: 0.7.6 - MDBOOK_MERMAID_VERSION: 0.11.2 - MDBOOK_TOC_VERSION: 0.9.0 + MDBOOK_MERMAID_VERSION: 0.12.6 + MDBOOK_TOC_VERSION: 0.11.2 DEPLOY_DIR: book/html BASE_SHA: ${{ github.event.pull_request.base.sha }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v3 with: @@ -45,7 +46,7 @@ jobs: if: github.event_name != 'push' run: | shopt -s globstar - MAX_LINE_LENGTH=100 bash ci/check_line_lengths.sh src/**/*.md + MAX_LINE_LENGTH=100 bash ci/lengthcheck.sh src/**/*.md - name: Install latest nightly Rust toolchain if: steps.mdbook-cache.outputs.cache-hit != 'true' diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..fee82ad8c --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,9 @@ +cff-version: 1.2.0 +message: If you use this guide, please cite it using these metadata. +title: Rust Compiler Development Guide (rustc-dev-guide) +abstract: A guide to developing the Rust compiler (rustc) +authors: + - name: "The Rust Project Developers" +date-released: "2018-01-16" +license: "MIT OR Apache-2.0" +repository-code: "https://github.com/rust-lang/rustc-dev-guide" diff --git a/README.md b/README.md index 006ea3231..131651e38 100644 --- a/README.md +++ b/README.md @@ -43,16 +43,16 @@ rustdocs][rustdocs]. To build a local static HTML site, install [`mdbook`](https://github.com/rust-lang/mdBook) with: ``` -> cargo install mdbook mdbook-linkcheck mdbook-toc +> cargo install mdbook mdbook-linkcheck mdbook-toc mdbook-mermaid ``` and execute the following command in the root of the repository: ``` -> mdbook build +> mdbook build --open ``` -The build files are found in the `book` directory. +The build files are found in the `book/html` directory. ### Link Validations @@ -67,20 +67,20 @@ including the `` marker at the place where you want the TOC. ### Pre-commit script We also test that line lengths are less than 100 columns. To test this locally, -you can run `ci/check_line_lengths.sh`. +you can run `ci/lengthcheck.sh`. You can also set this to run automatically. On Linux: ```bash -ln -s ../../ci/check_line_lengths.sh .git/hooks/pre-commit +ln -s ../../ci/lengthcheck.sh .git/hooks/pre-commit ``` On Windows: ```powershell -New-Item -Path .git/hooks/pre-commit -ItemType HardLink -Value +New-Item -Path .git/hooks/pre-commit -ItemType HardLink -Value $(Resolve-Path ci/lengthcheck.sh) ``` ## How to fix toolstate failures @@ -118,7 +118,7 @@ git submodule update --remote src/doc/rustc-dev-guide git add -u git commit -m "Update rustc-dev-guide" # Note that you can use -i, which is short for --incremental, in the following command -./x.py test --incremental src/doc/rustc-dev-guide # This is optional and should succeed anyway +./x test --incremental src/doc/rustc-dev-guide # This is optional and should succeed anyway # Open a PR in rust-lang/rust ``` diff --git a/book.toml b/book.toml index dc216760e..203bfd61e 100644 --- a/book.toml +++ b/book.toml @@ -1,7 +1,7 @@ [book] -title = "Guide to Rustc Development" -author = "Rustc developers" -description = "A guide to developing rustc" +title = "Rust Compiler Development Guide" +author = "The Rust Project Developers" +description = "A guide to developing the Rust compiler (rustc)" [build] create-missing = false @@ -18,6 +18,9 @@ git-repository-url = "https://github.com/rust-lang/rustc-dev-guide" edit-url-template = "https://github.com/rust-lang/rustc-dev-guide/edit/master/{path}" additional-js = ["mermaid.min.js", "mermaid-init.js"] +[output.html.search] +use-boolean-and = true + [output.html.fold] enable = true level = 0 @@ -42,5 +45,6 @@ warning-policy = "error" [output.html.redirect] "/compiletest.html" = "tests/compiletest.html" -"/diagnostics/sessiondiagnostic.html" = "diagnostics/diagnostic-structs.html" +"/diagnostics/sessiondiagnostic.html" = "diagnostic-structs.html" +"/diagnostics/diagnostic-codes.html" = "error-codes.html" "/miri.html" = "const-eval/interpret.html" diff --git a/ci/date-check/src/main.rs b/ci/date-check/src/main.rs index 70fce8b1c..d9e8145a3 100644 --- a/ci/date-check/src/main.rs +++ b/ci/date-check/src/main.rs @@ -170,7 +170,7 @@ fn main() { for (path, dates) in dates_by_file { println!( - "- [ ] {}", + "- {}", path.strip_prefix(&root_dir_path).unwrap_or(&path).display(), ); for (line, date) in dates { diff --git a/ci/check_line_lengths.sh b/ci/lengthcheck.sh similarity index 87% rename from ci/check_line_lengths.sh rename to ci/lengthcheck.sh index 31cda5c65..76d677be7 100755 --- a/ci/check_line_lengths.sh +++ b/ci/lengthcheck.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +# Check files for lines that are too long. + if [ "$1" == "--help" ]; then echo 'Usage:' "[MAX_LINE_LENGTH=n] $0 [file ...]" exit 1 @@ -10,8 +12,7 @@ if [ "$MAX_LINE_LENGTH" == "" ]; then fi if [ "$1" == "" ]; then - shopt -s globstar - files=( src/**/*.md ) + files=( src/*.md src/*/*.md src/*/*/*.md ) else files=( "$@" ) fi @@ -22,7 +23,6 @@ echo "Offending files and lines:" (( bad_lines = 0 )) (( inside_block = 0 )) for file in "${files[@]}"; do - echo "$file" (( line_no = 0 )) while IFS="" read -r line || [[ -n "$line" ]] ; do (( line_no++ )) @@ -34,7 +34,7 @@ for file in "${files[@]}"; do && ! [[ "$line" =~ " | "|"-|-"|"://"|"]:"|\[\^[^\ ]+\]: ]] \ && (( "${#line}" > $MAX_LINE_LENGTH )) ; then (( bad_lines++ )) - echo -e "\t$line_no : $line" + echo -e "\t$file:$line_no : $line" fi done < "$file" done diff --git a/ci/linkcheck.sh b/ci/linkcheck.sh index 5d49d1337..133e22239 100755 --- a/ci/linkcheck.sh +++ b/ci/linkcheck.sh @@ -3,12 +3,16 @@ set -e set -o pipefail +set_github_token() { + jq '.config.output.linkcheck."http-headers"."github\\.com" = ["Authorization: Bearer $GITHUB_TOKEN"]' +} + # https://docs.github.com/en/actions/reference/environment-variables if [ "$GITHUB_EVENT_NAME" = "schedule" ] ; then # running in scheduled job FLAGS="" + USE_TOKEN=1 echo "Doing full link check." - set -x elif [ "$GITHUB_EVENT_NAME" = "pull_request" ] ; then # running in PR CI build if [ -z "$BASE_SHA" ]; then echo "error: unexpected state: BASE_SHA must be non-empty in CI" @@ -17,9 +21,9 @@ elif [ "$GITHUB_EVENT_NAME" = "pull_request" ] ; then # running in PR CI build CHANGED_FILES=$(git diff --name-only $BASE_SHA... | tr '\n' ' ') FLAGS="--no-cache -f $CHANGED_FILES" + USE_TOKEN=1 echo "Checking files changed since $BASE_SHA: $CHANGED_FILES" - set -x else # running locally COMMIT_RANGE=master... CHANGED_FILES=$(git diff --name-only $COMMIT_RANGE | tr '\n' ' ') @@ -28,4 +32,10 @@ else # running locally echo "Checking files changed in $COMMIT_RANGE: $CHANGED_FILES" fi -exec mdbook-linkcheck $FLAGS +echo "exec mdbook-linkcheck $FLAGS" +if [ "$USE_TOKEN" = 1 ]; then + config=$(set_github_token) + exec mdbook-linkcheck $FLAGS <<<"$config" +else + exec mdbook-linkcheck $FLAGS +fi diff --git a/examples/README b/examples/README new file mode 100644 index 000000000..ca49dd74d --- /dev/null +++ b/examples/README @@ -0,0 +1,11 @@ +For each example to compile, you will need to first run the following: + + rustup component add rustc-dev llvm-tools + +To create an executable: + + rustc rustc-driver-example.rs + +To run an executable: + + rustup run nightly ./rustc-driver-example diff --git a/examples/rustc-driver-example.rs b/examples/rustc-driver-example.rs index 4203fe96a..fc0a5f469 100644 --- a/examples/rustc-driver-example.rs +++ b/examples/rustc-driver-example.rs @@ -1,10 +1,6 @@ #![feature(rustc_private)] -// NOTE: For the example to compile, you will need to first run the following: -// rustup component add rustc-dev llvm-tools-preview - -// version: 1.62.0-nightly (7c4b47696 2022-04-30) - +extern crate rustc_driver; extern crate rustc_error_codes; extern crate rustc_errors; extern crate rustc_hash; @@ -46,11 +42,10 @@ fn main() { "# .into(), }, - input_path: None, // Option output_dir: None, // Option output_file: None, // Option file_loader: None, // Option> - diagnostic_output: rustc_session::DiagnosticOutput::Default, + locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES, lint_caps: FxHashMap::default(), // FxHashMap // This is a callback from the driver that is called when [`ParseSess`] is created. parse_sess_created: None, //Option> @@ -68,21 +63,23 @@ fn main() { // Registry of diagnostics codes. registry: registry::Registry::new(&rustc_error_codes::DIAGNOSTICS), make_codegen_backend: None, + expanded_args: Vec::new(), + ice_file: None, }; rustc_interface::run_compiler(config, |compiler| { compiler.enter(|queries| { // Parse the program and print the syntax tree. - let parse = queries.parse().unwrap().take(); + let parse = queries.parse().unwrap().get_mut().clone(); println!("{parse:?}"); // Analyze the program and inspect the types of definitions. - queries.global_ctxt().unwrap().take().enter(|tcx| { + queries.global_ctxt().unwrap().enter(|tcx| { for id in tcx.hir().items() { let hir = tcx.hir(); let item = hir.item(id); match item.kind { rustc_hir::ItemKind::Static(_, _, _) | rustc_hir::ItemKind::Fn(_, _, _) => { let name = item.ident; - let ty = tcx.type_of(hir.local_def_id(item.hir_id())); + let ty = tcx.type_of(item.hir_id().owner.def_id); println!("{name:?}:\t{ty:?}") } _ => (), diff --git a/examples/rustc-driver-getting-diagnostics.rs b/examples/rustc-driver-getting-diagnostics.rs index 1d3a4034e..0195224de 100644 --- a/examples/rustc-driver-getting-diagnostics.rs +++ b/examples/rustc-driver-getting-diagnostics.rs @@ -1,10 +1,6 @@ #![feature(rustc_private)] -// NOTE: For the example to compile, you will need to first run the following: -// rustup component add rustc-dev llvm-tools-preview - -// version: 1.62.0-nightly (7c4b47696 2022-04-30) - +extern crate rustc_driver; extern crate rustc_error_codes; extern crate rustc_errors; extern crate rustc_hash; @@ -65,26 +61,24 @@ fn main() { " .into(), }, - // Redirect the diagnostic output of the compiler to a buffer. - diagnostic_output: rustc_session::DiagnosticOutput::Raw(Box::from(DiagnosticSink( - buffer.clone(), - ))), crate_cfg: rustc_hash::FxHashSet::default(), crate_check_cfg: CheckCfg::default(), - input_path: None, output_dir: None, output_file: None, file_loader: None, + locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES, lint_caps: rustc_hash::FxHashMap::default(), parse_sess_created: None, register_lints: None, override_queries: None, registry: registry::Registry::new(&rustc_error_codes::DIAGNOSTICS), make_codegen_backend: None, + expanded_args: Vec::new(), + ice_file: None, }; rustc_interface::run_compiler(config, |compiler| { compiler.enter(|queries| { - queries.global_ctxt().unwrap().take().enter(|tcx| { + queries.global_ctxt().unwrap().enter(|tcx| { // Run the analysis phase on the local crate to trigger the type error. let _ = tcx.analysis(()); }); diff --git a/examples/rustc-driver-interacting-with-the-ast.rs b/examples/rustc-driver-interacting-with-the-ast.rs index 231994a97..3d3827e5d 100644 --- a/examples/rustc-driver-interacting-with-the-ast.rs +++ b/examples/rustc-driver-interacting-with-the-ast.rs @@ -1,11 +1,7 @@ #![feature(rustc_private)] -// NOTE: For the example to compile, you will need to first run the following: -// rustup component add rustc-dev llvm-tools-preview - -// version: 1.62.0-nightly (7c4b47696 2022-04-30) - extern crate rustc_ast_pretty; +extern crate rustc_driver; extern crate rustc_error_codes; extern crate rustc_errors; extern crate rustc_hash; @@ -43,30 +39,30 @@ fn main() { "# .to_string(), }, - diagnostic_output: rustc_session::DiagnosticOutput::Default, crate_cfg: rustc_hash::FxHashSet::default(), crate_check_cfg: CheckCfg::default(), - input_path: None, output_dir: None, output_file: None, file_loader: None, + locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES, lint_caps: rustc_hash::FxHashMap::default(), parse_sess_created: None, register_lints: None, override_queries: None, make_codegen_backend: None, registry: registry::Registry::new(&rustc_error_codes::DIAGNOSTICS), + expanded_args: Vec::new(), + ice_file: None, }; rustc_interface::run_compiler(config, |compiler| { compiler.enter(|queries| { // TODO: add this to -Z unpretty - let ast_krate = queries.parse().unwrap().take(); + let ast_krate = queries.parse().unwrap().get_mut().clone(); for item in ast_krate.items { println!("{}", item_to_string(&item)); } - // Analyze the crate and inspect the types under the cursor. - queries.global_ctxt().unwrap().take().enter(|tcx| { + queries.global_ctxt().unwrap().enter(|tcx| { // Every compilation contains a single crate. let hir_krate = tcx.hir(); // Iterate over the top-level items in the crate, looking for the main function. @@ -79,7 +75,7 @@ fn main() { if let rustc_hir::StmtKind::Local(local) = block.stmts[0].kind { if let Some(expr) = local.init { let hir_id = expr.hir_id; // hir_id identifies the string "Hello, world!" - let def_id = tcx.hir().local_def_id(item.hir_id()); // def_id identifies the main function + let def_id = item.hir_id().owner.def_id; // def_id identifies the main function let ty = tcx.typeck(def_id).node_type(hir_id); println!("{expr:#?}: {ty:?}"); } diff --git a/src/SUMMARY.md b/src/SUMMARY.md index 8943984aa..bfa9325d6 100644 --- a/src/SUMMARY.md +++ b/src/SUMMARY.md @@ -1,20 +1,20 @@ # Summary -[About this guide](./about-this-guide.md) - [Getting Started](./getting-started.md) +[About this guide](./about-this-guide.md) --- # Building and debugging `rustc` -- [How to Build and Run the Compiler](./building/how-to-build-and-run.md) +- [How to build and run the compiler](./building/how-to-build-and-run.md) - [Prerequisites](./building/prerequisites.md) - [Suggested Workflows](./building/suggested.md) - [Distribution artifacts](./building/build-install-distribution-artifacts.md) - [Building Documentation](./building/compiler-documenting.md) - [Rustdoc overview](./rustdoc.md) - [Adding a new target](./building/new-target.md) + - [Optimized build](./building/optimized-build.md) - [Testing the compiler](./tests/intro.md) - [Running tests](./tests/running.md) - [Testing with Docker](./tests/docker.md) @@ -25,7 +25,8 @@ - [Test headers](./tests/headers.md) - [Performance testing](./tests/perf.md) - [Crater](./tests/crater.md) -- [Debugging the Compiler](./compiler-debugging.md) + - [Suggest tests tool](./tests/suggest-tests.md) +- [Debugging the compiler](./compiler-debugging.md) - [Using the tracing/logging instrumentation](./tracing.md) - [Profiling the compiler](./profiling.md) - [with the linux perf tool](./profiling/with_perf.md) @@ -35,17 +36,19 @@ # Contributing to Rust -- [Introduction](./contributing.md) +- [Contribution Procedures](./contributing.md) - [About the compiler team](./compiler-team.md) - [Using Git](./git.md) - [Mastering @rustbot](./rustbot.md) - [Walkthrough: a typical contribution](./walkthrough.md) -- [Bug Fix Procedure](./bug-fix-procedure.md) -- [Implementing new features](./implementing_new_features.md) +- [Implementing new language features](./implementing_new_features.md) - [Stability attributes](./stability.md) - [Stabilizing Features](./stabilization_guide.md) - [Feature Gates](./feature-gates.md) - [Coding conventions](./conventions.md) +- [Procedures for Breaking Changes](./bug-fix-procedure.md) +- [Using external repositories](./external-repos.md) +- [Fuzzing](./fuzzing.md) - [Notification groups](notification-groups/about.md) - [ARM](notification-groups/arm.md) - [Cleanup Crew](notification-groups/cleanup-crew.md) @@ -57,7 +60,7 @@ # High-level Compiler Architecture - [Prologue](./part-2-intro.md) -- [Overview of the Compiler](./overview.md) +- [Overview of the compiler](./overview.md) - [The compiler source code](./compiler-src.md) - [Bootstrapping](./building/bootstrapping.md) - [Queries: demand-driven compilation](./query.md) @@ -75,7 +78,7 @@ - [Prologue](./part-3-intro.md) - [Command-line arguments](./cli.md) -- [The Rustc Driver and Interface](./rustc-driver.md) +- [rustc_driver and rustc_interface](./rustc-driver.md) - [Example: Type checking](./rustc-driver-interacting-with-the-ast.md) - [Example: Getting diagnostics](./rustc-driver-getting-diagnostics.md) - [Syntax and the AST](./syntax-intro.md) @@ -95,7 +98,7 @@ - [MIR construction](./mir/construction.md) - [MIR visitor and traversal](./mir/visitor.md) - [MIR passes: getting the MIR for a function](./mir/passes.md) -- [Identifiers in the Compiler](./identifiers.md) +- [Identifiers in the compiler](./identifiers.md) - [Closure expansion](./closure.md) - [Inline assembly](./asm.md) @@ -107,9 +110,12 @@ - [`TypeFolder` and `TypeFoldable`](./ty-fold.md) - [Generic arguments](./generic_arguments.md) - [Constants in the type system](./constants.md) + - [Bound vars and Parameters](./bound-vars-and-params.md) - [Type inference](./type-inference.md) - [Trait solving](./traits/resolution.md) - - [Early and Late Bound Parameters](./early-late-bound.md) + - [Early and Late Bound Parameter Definitions](./early-late-bound-summary.md) + - [What are early and late bound parameters](./what-does-early-late-bound-mean.md) + - [Interactions with turbofishing](./turbofishing-and-early-late-bound.md) - [Higher-ranked trait bounds](./traits/hrtb.md) - [Caching subtleties](./traits/caching.md) - [Specialization](./traits/specialization.md) @@ -117,12 +123,22 @@ - [Lowering to logic](./traits/lowering-to-logic.md) - [Goals and clauses](./traits/goals-and-clauses.md) - [Canonical queries](./traits/canonical-queries.md) - - [Canonicalization](./traits/canonicalization.md) + - [Canonicalization](./traits/canonicalization.md) + - [Next-gen trait solving](./solve/trait-solving.md) + - [Invariants of the type system](./solve/invariants.md) + - [The solver](./solve/the-solver.md) + - [Canonicalization](./solve/canonicalization.md) + - [Coinduction](./solve/coinduction.md) + - [Proof trees](./solve/proof-trees.md) + - [Normalization](./solve/normalization.md) + - [`Unsize` and `CoerceUnsized` traits](./traits/unsize.md) - [Type checking](./type-checking.md) - [Method Lookup](./method-lookup.md) - [Variance](./variance.md) - [Opaque Types](./opaque-types-type-alias-impl-trait.md) - [Inference details](./opaque-types-impl-trait-inference.md) + - [Return Position Impl Trait In Trait](./return-position-impl-trait-in-trait.md) +- [Effect checking](./effects.md) - [Pattern and Exhaustiveness Checking](./pat-exhaustive-checking.md) - [MIR dataflow](./mir/dataflow.md) - [Drop elaboration](./mir/drop-elaboration.md) @@ -130,6 +146,7 @@ - [Tracking moves and initialization](./borrow_check/moves_and_initialization.md) - [Move paths](./borrow_check/moves_and_initialization/move_paths.md) - [MIR type checker](./borrow_check/type_check.md) + - [Drop check](./borrow_check/drop_check.md) - [Region inference](./borrow_check/region_inference.md) - [Constraint propagation](./borrow_check/region_inference/constraint_propagation.md) - [Lifetime parameters](./borrow_check/region_inference/lifetime_parameters.md) @@ -143,7 +160,7 @@ - [Diagnostic and subdiagnostic structs](./diagnostics/diagnostic-structs.md) - [Translation](./diagnostics/translation.md) - [`LintStore`](./diagnostics/lintstore.md) - - [Diagnostic codes](./diagnostics/diagnostic-codes.md) + - [Error codes](./diagnostics/error-codes.md) - [Diagnostic items](./diagnostics/diagnostic-items.md) - [`ErrorGuaranteed`](./diagnostics/error-guaranteed.md) @@ -165,7 +182,7 @@ - [Profile-guided Optimization](./profile-guided-optimization.md) - [LLVM Source-Based Code Coverage](./llvm-coverage-instrumentation.md) - [Sanitizers Support](./sanitizers.md) -- [Debugging Support in the Rust Compiler](./debugging-support-in-rustc.md) +- [Debugging support in the Rust compiler](./debugging-support-in-rustc.md) --- diff --git a/src/about-this-guide.md b/src/about-this-guide.md index 71407854e..f43f86e5f 100644 --- a/src/about-this-guide.md +++ b/src/about-this-guide.md @@ -58,14 +58,50 @@ please see the corresponding [subsection on writing documentation in this guide] You might also find the following sites useful: -- [rustc API docs] -- rustdoc documentation for the compiler +- This guide contains information about how various parts of the + compiler work and how to contribute to the compiler. +- [rustc API docs] -- rustdoc documentation for the compiler, devtools, and internal tools - [Forge] -- contains documentation about Rust infrastructure, team procedures, and more - [compiler-team] -- the home-base for the Rust compiler team, with description of the team procedures, active working groups, and the team calendar. - [std-dev-guide] -- a similar guide for developing the standard library. +- [The t-compiler zulip][z] +- `#contribute` and `#wg-rustup` on [Discord](https://discord.gg/rust-lang). +- The [Rust Internals forum][rif], a place to ask questions and + discuss Rust's internals +- The [Rust reference][rr], even though it doesn't specifically talk about + Rust's internals, is a great resource nonetheless +- Although out of date, [Tom Lee's great blog article][tlgba] is very helpful +- [rustaceans.org][ro] is helpful, but mostly dedicated to IRC +- The [Rust Compiler Testing Docs][rctd] +- For [@bors], [this cheat sheet][cheatsheet] is helpful +- Google is always helpful when programming. + You can [search all Rust documentation][gsearchdocs] (the standard library, + the compiler, the books, the references, and the guides) to quickly find + information about the language and compiler. +- You can also use Rustdoc's built-in search feature to find documentation on + types and functions within the crates you're looking at. You can also search + by type signature! For example, searching for `* -> vec` should find all + functions that return a `Vec`. + _Hint:_ Find more tips and keyboard shortcuts by typing `?` on any Rustdoc + page! + +[rustc dev guide]: about-this-guide.md +[gsearchdocs]: https://www.google.com/search?q=site:doc.rust-lang.org+your+query+here +[stddocs]: https://doc.rust-lang.org/std +[rif]: http://internals.rust-lang.org +[rr]: https://doc.rust-lang.org/book/README.html +[rustforge]: https://forge.rust-lang.org/ +[tlgba]: https://tomlee.co/2014/04/a-more-detailed-tour-of-the-rust-compiler/ +[ro]: https://www.rustaceans.org/ +[rctd]: tests/intro.md +[cheatsheet]: https://bors.rust-lang.org/ +[Miri]: https://github.com/rust-lang/miri +[@bors]: https://github.com/bors [GitHub repository]: https://github.com/rust-lang/rustc-dev-guide/ -[rustc API docs]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ +[rustc API docs]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle [Forge]: https://forge.rust-lang.org/ [compiler-team]: https://github.com/rust-lang/compiler-team/ [std-dev-guide]: https://std-dev-guide.rust-lang.org/ +[z]: https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler diff --git a/src/appendix/background.md b/src/appendix/background.md index 0b83b010f..05d01be0f 100644 --- a/src/appendix/background.md +++ b/src/appendix/background.md @@ -300,7 +300,7 @@ does not. > Thanks to `mem`, `scottmcm`, and `Levi` on the official Discord for the > recommendations, and to `tinaun` for posting a link to a [twitter thread from -> Graydon Hoare](https://twitter.com/graydon_pub/status/1039615569132118016) +> Graydon Hoare](https://web.archive.org/web/20181230012554/https://twitter.com/graydon_pub/status/1039615569132118016) > which had some more recommendations! > > Other sources: https://gcc.gnu.org/wiki/ListOfCompilerBooks diff --git a/src/appendix/bibliography.md b/src/appendix/bibliography.md index c2bb00e3b..cc262abef 100644 --- a/src/appendix/bibliography.md +++ b/src/appendix/bibliography.md @@ -79,7 +79,7 @@ Rust, as well as publications about Rust. Rust](https://munksgaard.me/papers/laumann-munksgaard-larsen.pdf). Philip Munksgaard's master's thesis. Research for Servo. * [Ownership is Theft: Experiences Building an Embedded OS in Rust - Amit Levy, et. al.](https://amitlevy.com/papers/tock-plos2015.pdf) -* [You can't spell trust without Rust](https://raw.githubusercontent.com/Gankro/thesis/master/thesis.pdf). Alexis Beingessner's master's thesis. +* [You can't spell trust without Rust](https://faultlore.com/blah/papers/thesis.pdf). Aria Beingessner's master's thesis. * [Rust-Bio: a fast and safe bioinformatics library](https://academic.oup.com/bioinformatics/article/32/3/444/1743419). Johannes Köster * [Safe, Correct, and Fast Low-Level Networking](https://octarineparrot.com/assets/msci_paper.pdf). Robert Clipsham's master's thesis. * [Formalizing Rust traits](https://open.library.ubc.ca/cIRcle/collections/ubctheses/24/items/1.0220521). Jonatan Milewski's master's thesis. diff --git a/src/appendix/glossary.md b/src/appendix/glossary.md index 42306dc1c..ee3a3a720 100644 --- a/src/appendix/glossary.md +++ b/src/appendix/glossary.md @@ -5,6 +5,7 @@ Term | Meaning ------------------------------------------------------|-------- arena/arena allocation   | An _arena_ is a large memory buffer from which other memory allocations are made. This style of allocation is called _arena allocation_. See [this chapter](../memory.md) for more info. AST   | The abstract syntax tree produced by the `rustc_ast` crate; reflects user syntax very closely. +APIT   | An argument-position `impl Trait`. Also known as an anonymous type parameter. ([see the reference](https://doc.rust-lang.org/reference/types/impl-trait.html#anonymous-type-parameters)). binder   | A "binder" is a place where a variable or type is declared; for example, the `` is a binder for the generic type parameter `T` in `fn foo(..)`, and \|`a`\|` ...` is a binder for the parameter `a`. See [the background chapter for more](./background.md#free-vs-bound). BodyId   | An identifier that refers to a specific body (definition of a function or constant) in the crate. See [the HIR chapter for more](../hir.md#identifiers-in-the-hir). bound variable   | A "bound variable" is one that is declared within an expression/term. For example, the variable `a` is bound within the closure expression \|`a`\|` a * 2`. See [the background chapter for more](./background.md#free-vs-bound) @@ -23,7 +24,8 @@ Term | Meaning double pointer   | A pointer with additional metadata. See "fat pointer" for more. drop glue   | (internal) compiler-generated instructions that handle calling the destructors (`Drop`) for data types. DST   | Short for Dynamically-Sized Type, this is a type for which the compiler cannot statically know the size in memory (e.g. `str` or `[u8]`). Such types don't implement `Sized` and cannot be allocated on the stack. They can only occur as the last field in a struct. They can only be used behind a pointer (e.g. `&str` or `&[u8]`). -early-bound lifetime   | A lifetime region that is substituted at its definition site. Bound in an item's `Generics` and substituted using a `Substs`. Contrast with **late-bound lifetime**. ([see more](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.RegionKind.html#bound-regions)) +early-bound lifetime   | A lifetime region that is substituted at its definition site. Bound in an item's `Generics` and substituted using a `GenericArgs`. Contrast with **late-bound lifetime**. ([see more](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.RegionKind.html#bound-regions)) +effects   | Right now only means const traits and `~const` bounds. ([see more](../effects.md)) empty type   | see "uninhabited type". fat pointer   | A two word value carrying the address of some value, along with some further information necessary to put the value to use. Rust includes two kinds of "fat pointers": references to slices, and trait objects. A reference to a slice carries the starting address of the slice and its length. A trait object carries a value's address and a pointer to the trait's implementation appropriate to that value. "Fat pointers" are also known as "wide pointers", and "double pointers". free variable   | A "free variable" is one that is not bound within an expression or term; see [the background chapter for more](./background.md#free-vs-bound) @@ -42,7 +44,7 @@ Term | Meaning IRLO   | `IRLO` or `irlo` is sometimes used as an abbreviation for [internals.rust-lang.org](https://internals.rust-lang.org). item   | A kind of "definition" in the language, such as a static, const, use statement, module, struct, etc. Concretely, this corresponds to the `Item` type. lang item   | Items that represent concepts intrinsic to the language itself, such as special built-in traits like `Sync` and `Send`; or traits representing operations such as `Add`; or functions that are called by the compiler. ([see more](https://doc.rust-lang.org/1.9.0/book/lang-items.html)) -late-bound lifetime   | A lifetime region that is substituted at its call site. Bound in a HRTB and substituted by specific functions in the compiler, such as `liberate_late_bound_regions`. Contrast with **early-bound lifetime**. ([see more](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.RegionKind.html#bound-regions)) +late-bound lifetime   | A lifetime region that is substituted at its call site. Bound in a HRTB and substituted by specific functions in the compiler, such as `liberate_late_bound_regions`. Contrast with **early-bound lifetime**. ([see more](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.RegionKind.html#bound-regions)) local crate   | The crate currently being compiled. This is in contrast to "upstream crates" which refer to dependencies of the local crate. LTO   | Short for Link-Time Optimizations, this is a set of optimizations offered by LLVM that occur just before the final binary is linked. These include optimizations like removing functions that are never used in the final program, for example. _ThinLTO_ is a variant of LTO that aims to be a bit more scalable and efficient, but possibly sacrifices some optimizations. You may also read issues in the Rust repo about "FatLTO", which is the loving nickname given to non-Thin LTO. LLVM documentation: [here][lto] and [here][thinlto]. [LLVM]   | (actually not an acronym :P) an open-source compiler backend. It accepts LLVM IR and outputs native binaries. Various languages (e.g. Rust) can then implement a compiler front-end that outputs LLVM IR and use LLVM to compile to all the platforms LLVM supports. @@ -67,6 +69,8 @@ Term | Meaning recovery   | Recovery refers to handling invalid syntax during parsing (e.g. a missing comma) and continuing to parse the AST. This avoid showing spurious errors to the user (e.g. showing 'missing field' errors when the struct definition contains errors). region   | Another term for "lifetime" often used in the literature and in the borrow checker. rib   | A data structure in the name resolver that keeps track of a single scope for names. ([see more](../name-resolution.md)) +RPIT   | A return-position `impl Trait`. ([see the reference](https://doc.rust-lang.org/reference/types/impl-trait.html#abstract-return-types)). +RPITIT   | A return-position `impl Trait` *in trait*. Unlike RPIT, this is desugared to a generic associated type (GAT). Introduced in [RFC 3425](https://rust-lang.github.io/rfcs/3425-return-position-impl-trait-in-traits.html). ([see more](../return-position-impl-trait-in-trait.md)) scrutinee   | A scrutinee is the expression that is matched on in `match` expressions and similar pattern matching constructs. For example, in `match x { A => 1, B => 2 }`, the expression `x` is the scrutinee. sess   | The compiler session, which stores global data used throughout compilation side tables   | Because the AST and HIR are immutable once created, we often carry extra information about them in the form of hashtables, indexed by the id of a particular node. @@ -76,6 +80,7 @@ Term | Meaning substs   | The substitutions for a given generic type or item (e.g. the `i32`, `u32` in `HashMap`). sysroot   | The directory for build artifacts that are loaded by the compiler at runtime. ([see more](../building/bootstrapping.html#what-is-a-sysroot)) tag   | The "tag" of an enum/generator encodes the [discriminant](#discriminant) of the active variant/state. Tags can either be "direct" (simply storing the discriminant in a field) or use a ["niche"](#niche). +TAIT   | A type-alias `impl Trait`. Introduced in [RFC 2515](https://rust-lang.github.io/rfcs/2515-type_alias_impl_trait.html). `tcx`   | Standard variable name for the "typing context" (`TyCtxt`), main data structure of the compiler. ([see more](../ty.md)) `'tcx`   | The lifetime of the allocation arenas used by `TyCtxt`. Most data interned during a compilation session will use this lifetime with the exception of HIR data which uses the `'hir` lifetime. ([see more](../ty.md)) token   | The smallest unit of parsing. Tokens are produced after lexing ([see more](../the-parser.md)). @@ -89,6 +94,7 @@ Term | Meaning upvar   | A variable captured by a closure from outside the closure. variance   | Determines how changes to a generic type/lifetime parameter affect subtyping; for example, if `T` is a subtype of `U`, then `Vec` is a subtype `Vec` because `Vec` is *covariant* in its generic parameter. See [the background chapter](./background.md#variance) for a more general explanation. See the [variance chapter](../variance.md) for an explanation of how type checking handles variance. variant index   | In an enum, identifies a variant by assigning them indices starting at 0. This is purely internal and not to be confused with the ["discriminant"](#discriminant) which can be overwritten by the user (e.g. `enum Bool { True = 42, False = 0 }`). +Well-formedness   | Semantically:An expression that evaluates to meaningful result. In Type Systems: A type related construct which follows rules of the type system. wide pointer   | A pointer with additional metadata. See "fat pointer" for more. ZST   | Zero-Sized Type. A type whose values have size 0 bytes. Since `2^0 = 1`, such types can have exactly one value. For example, `()` (unit) is a ZST. `struct Foo;` is also a ZST. The compiler can do some nice optimizations around ZSTs. diff --git a/src/appendix/humorust.md b/src/appendix/humorust.md index bc8255439..1b199d98a 100644 --- a/src/appendix/humorust.md +++ b/src/appendix/humorust.md @@ -3,7 +3,7 @@ What's a project without a sense of humor? And frankly some of these are enlightening? -- [Weird exprs test](https://github.com/rust-lang/rust/blob/master/src/test/ui/weird-exprs.rs) +- [Weird exprs test](https://github.com/rust-lang/rust/blob/master/tests/ui/weird-exprs.rs) - [Ferris Rap](http://fitzgeraldnick.com/2018/12/13/rust-raps.html) - [The Genesis of Generic Germination](https://github.com/rust-lang/rust/pull/53645#issue-210543221) - [The Bastion of the Turbofish test](https://github.com/rust-lang/rust/blob/79d8a0fcefa5134db2a94739b1d18daa01fc6e9f/src/test/ui/bastion-of-the-turbofish.rs) diff --git a/src/asm.md b/src/asm.md index 9b5a82f34..b19f2ad46 100644 --- a/src/asm.md +++ b/src/asm.md @@ -277,7 +277,7 @@ pub enum InlineAsmOperand<'tcx> { } ``` -As part of HAIR lowering, `InOut` and `SplitInOut` operands are lowered to a split form with a +As part of THIR lowering, `InOut` and `SplitInOut` operands are lowered to a split form with a separate `in_value` and `out_place`. Semantically, the `InlineAsm` terminator is similar to the `Call` terminator except that it has @@ -361,9 +361,9 @@ can't know ahead of time whether a function will require a frame/base pointer. Various tests for inline assembly are available: -- `src/test/assembly/asm` -- `src/test/ui/asm` -- `src/test/codegen/asm-*` +- `tests/assembly/asm` +- `tests/ui/asm` +- `tests/codegen/asm-*` Every architecture supported by inline assembly must have exhaustive tests in -`src/test/assembly/asm` which test all combinations of register classes and types. +`tests/assembly/asm` which test all combinations of register classes and types. diff --git a/src/ast-validation.md b/src/ast-validation.md index ff549ca62..f3ac35e8a 100644 --- a/src/ast-validation.md +++ b/src/ast-validation.md @@ -1,6 +1,36 @@ # AST Validation -AST validation is the process of checking various correctness properties about -the AST after macro expansion. + -**TODO**: write this chapter. [#656](https://github.com/rust-lang/rustc-dev-guide/issues/656) +## About + +_AST validation_ is a separate AST pass that visits each +item in the tree and performs simple checks. This pass +doesn't perform any complex analysis, type checking or +name resolution. + +Before performing any validation, the compiler first expands +the macros. Then this pass performs validations to check +that each AST item is in the correct state. And when this pass +is done, the compiler runs the crate resolution pass. + +## Validations + +Validations are defined in `AstValidator` type, which +itself is located in `rustc_ast_passes` crate. This +type implements various simple checks which emit errors +when certain language rules are broken. + +In addition, `AstValidator` implements `Visitor` trait +that defines how to visit AST items (which can be functions, +traits, enums, etc). + +For each item, visitor performs specific checks. For +example, when visiting a function declaration, +`AstValidator` checks that the function has: + +* no more than `u16::MAX` parameters; +* c-variadic functions are declared with at least one named argument; +* c-variadic argument goes the last in the declaration; +* documentation comments aren't applied to function parameters; +* and other validations. diff --git a/src/backend/backend-agnostic.md b/src/backend/backend-agnostic.md index ea50cd754..c5eb57e64 100644 --- a/src/backend/backend-agnostic.md +++ b/src/backend/backend-agnostic.md @@ -2,16 +2,21 @@ -As of Aug 2022, `rustc_codegen_ssa` provides an -abstract interface for all backends to implement, to allow other codegen -backends (e.g. [Cranelift]). +[`rustc_codegen_ssa`] +provides an abstract interface for all backends to implement, +namely LLVM, [Cranelift], and [GCC]. -[Cranelift]: https://github.com/bytecodealliance/wasmtime/tree/HEAD/cranelift +[Cranelift]: https://github.com/bjorn3/rustc_codegen_cranelift +[GCC]: https://github.com/rust-lang/rustc_codegen_gcc +[`rustc_codegen_ssa`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/index.html -# Refactoring of `rustc_codegen_llvm` +Below is some background information on the refactoring that created this +abstract interface. + +## Refactoring of `rustc_codegen_llvm` by Denis Merigoux, October 23rd 2018 -## State of the code before the refactoring +### State of the code before the refactoring All the code related to the compilation of MIR into LLVM IR was contained inside the `rustc_codegen_llvm` crate. Here is the breakdown of the most @@ -51,7 +56,7 @@ While the LLVM-specific code will be left in `rustc_codegen_llvm`, all the new traits and backend-agnostic code will be moved in `rustc_codegen_ssa` (name suggestion by @eddyb). -## Generic types and structures +### Generic types and structures @irinagpopa started to parametrize the types of `rustc_codegen_llvm` by a generic `Value` type, implemented in LLVM by a reference `&'ll Value`. This @@ -103,7 +108,7 @@ of the backend and it makes more sense to leave their definition to the backend implementor than to allow just a narrow spot via a generic field for the backend's context. -## Traits and interface +### Traits and interface Because they have to be defined by the backend, `CodegenCx` and `Builder` will be the structures implementing all the traits defining the backend's interface. @@ -170,7 +175,7 @@ called. However, when implementing a Rust backend for `rustc`, these methods will need information from `CodegenCx`, hence the additional parameter (unused in the LLVM implementation of the trait). -## State of the code after the refactoring +### State of the code after the refactoring The traits offer an API which is very similar to the API of LLVM. This is not the best solution since LLVM has a very special way of doing things: when diff --git a/src/backend/implicit-caller-location.md b/src/backend/implicit-caller-location.md index 21554f5a4..cf100a2e5 100644 --- a/src/backend/implicit-caller-location.md +++ b/src/backend/implicit-caller-location.md @@ -260,7 +260,7 @@ originally specified. They have since been implemented following the path which to the author and reviewers. [RFC 2091]: https://github.com/rust-lang/rfcs/blob/master/text/2091-inline-semantic.md -[attr-reference]: https://doc.rust-lang.org/reference/attributes/diagnostics.html#the-track_caller-attribute +[attr-reference]: https://doc.rust-lang.org/reference/attributes/codegen.html#the-track_caller-attribute [intrinsic]: https://doc.rust-lang.org/nightly/core/intrinsics/fn.caller_location.html [wrapper]: https://doc.rust-lang.org/nightly/core/panic/struct.Location.html#method.caller [non-viable alternatives]: https://github.com/rust-lang/rfcs/blob/master/text/2091-inline-semantic.md#non-viable-alternatives diff --git a/src/backend/libs-and-metadata.md b/src/backend/libs-and-metadata.md index 5e005c965..b92a40b8b 100644 --- a/src/backend/libs-and-metadata.md +++ b/src/backend/libs-and-metadata.md @@ -103,9 +103,8 @@ The hash includes a variety of elements: * Hashes of the HIR nodes. * All of the upstream crate hashes. * All of the source filenames. -* Hashes of certain command-line flags (like `-C metadata` via the [Crate - Disambiguator](#crate-disambiguator), and all CLI options marked with - `[TRACKED]`). +* Hashes of certain command-line flags (like `-C metadata` via the [Stable + Crate Id](#stable-crate-id), and all CLI options marked with `[TRACKED]`). See [`compute_hir_hash`] for where the hash is actually computed. diff --git a/src/backend/monomorph.md b/src/backend/monomorph.md index 4a0b55f62..7726daf4f 100644 --- a/src/backend/monomorph.md +++ b/src/backend/monomorph.md @@ -99,7 +99,7 @@ are relatively rare in functions, but closures inherit the generic parameters of their parent function and it is common for closures to not use those inherited parameters. Without polymorphization, a copy of these closures would be created for each copy of the parent function. By -creating fewer copies, less LLVM IR is generated and needs processed. +creating fewer copies, less LLVM IR is generated; therefore less needs to be processed. `unused_generic_params` returns a `FiniteBitSet` where a bit is set if the generic parameter of the corresponding index is unused. Any parameters @@ -133,7 +133,7 @@ the substitutions with the identity parameter before being added to the set of collected items - thereby reducing the copies from two (`[u16, u32]` and `[u64, u32]`) to one (`[A, u32]`). -`unused_generic_params` will also invoked during code generation when the +`unused_generic_params` will also be invoked during code generation when the symbol name for `foo` is being computed for use in the callsites of `foo` (which have the regular substitutions present, otherwise there would be a symbol mismatch between the caller and the function). diff --git a/src/backend/updating-llvm.md b/src/backend/updating-llvm.md index 38fbb2e44..dd16b1e28 100644 --- a/src/backend/updating-llvm.md +++ b/src/backend/updating-llvm.md @@ -2,7 +2,7 @@ - + There is no formal policy about when to update LLVM or what it can be updated to, but a few guidelines are applied: @@ -16,9 +16,9 @@ but a few guidelines are applied: There are two reasons we would want to update LLVM: -* A bug could have been fixed! Often we find bugs in the compiler and fix - them upstream in LLVM. We'll want to pull fixes back to the compiler itself as - they're merged upstream. +* A bug could have been fixed! + Note that if we are the ones who fixed such a bug, + we prefer to upstream it, then pull it back for use by rustc. * LLVM itself may have a new release. @@ -52,7 +52,7 @@ An example PR: ## New LLVM Release Updates - + Unlike bugfixes, updating to a new release of LLVM typically requires a lot more work. @@ -90,16 +90,16 @@ so let's go through each in detail. `src/llvm-project` to ensure submodule updates aren't reverted. Some commands you should execute are: - * `./x.py build src/llvm` - test that LLVM still builds - * `./x.py build src/tools/lld` - same for LLD - * `./x.py build` - build the rest of rustc + * `./x build src/llvm` - test that LLVM still builds + * `./x build src/tools/lld` - same for LLD + * `./x build` - build the rest of rustc You'll likely need to update [`llvm-wrapper/*.cpp`][`llvm-wrapper`] to compile with updated LLVM bindings. Note that you should use `#ifdef` and such to ensure that the bindings still compile on older LLVM versions. - Note that `profile = "compiler"` and other defaults set by `./x.py setup` + Note that `profile = "compiler"` and other defaults set by `./x setup` download LLVM from CI instead of building it from source. You should disable this temporarily to make sure your changes are being used. This is done by having the following setting in `config.toml`: @@ -139,6 +139,7 @@ so let's go through each in detail. > - [LLVM 13](https://github.com/rust-lang/rust/pull/87570) > - [LLVM 14](https://github.com/rust-lang/rust/pull/93577) > - [LLVM 15](https://github.com/rust-lang/rust/pull/99464) + > - [LLVM 16](https://github.com/rust-lang/rust/pull/109474) Note that sometimes it's easiest to land [`llvm-wrapper`] compatibility as a PR before actually updating `src/llvm-project`. @@ -172,9 +173,11 @@ so let's go through each in detail. Ideally the above instructions are pretty smooth, but here's some caveats to keep in mind while going through them: -* LLVM bugs are hard to find, don't hesitate to ask for help! Bisection is - definitely your friend here (yes LLVM takes forever to build, yet bisection is - still your friend) +* LLVM bugs are hard to find, don't hesitate to ask for help! + Bisection is definitely your friend here + (yes LLVM takes forever to build, yet bisection is still your friend). + Note that you can make use of [Dev Desktops], + which is an initiative to provide the contributors with remote access to powerful hardware. * If you've got general questions, [wg-llvm] can help you out. * Creating branches is a privileged operation on GitHub, so you'll need someone with write access to create the branches for you most likely. @@ -184,3 +187,4 @@ keep in mind while going through them: [llvm/llvm-project repository]: https://github.com/llvm/llvm-project [`llvm-wrapper`]: https://github.com/rust-lang/rust/tree/master/compiler/rustc_llvm/llvm-wrapper [wg-llvm]: https://rust-lang.zulipchat.com/#narrow/stream/187780-t-compiler.2Fwg-llvm +[Dev Desktops]: https://forge.rust-lang.org/infra/docs/dev-desktop.html diff --git a/src/borrow_check.md b/src/borrow_check.md index 8e2bb752a..f206da42a 100644 --- a/src/borrow_check.md +++ b/src/borrow_check.md @@ -27,7 +27,7 @@ HIR. Doing borrow checking on MIR has several advantages: ### Major phases of the borrow checker The borrow checker source is found in -[the `rustc_borrow_ck` crate][b_c]. The main entry point is +[the `rustc_borrowck` crate][b_c]. The main entry point is the [`mir_borrowck`] query. [b_c]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_borrowck/index.html diff --git a/src/borrow_check/drop_check.md b/src/borrow_check/drop_check.md new file mode 100644 index 000000000..51d1ddd83 --- /dev/null +++ b/src/borrow_check/drop_check.md @@ -0,0 +1,154 @@ +# Drop Check + +We generally require the type of locals to be well-formed whenever the +local is used. This includes proving the where-bounds of the local and +also requires all regions used by it to be live. + +The only exception to this is when implicitly dropping values when they +go out of scope. This does not necessarily require the value to be live: + +```rust +fn main() { + let x = vec![]; + { + let y = String::from("I am temporary"); + x.push(&y); + } + // `x` goes out of scope here, after the reference to `y` + // is invalidated. This means that while dropping `x` its type + // is not well-formed as it contain regions which are not live. +} +``` + +This is only sound if dropping the value does not try to access any dead +region. We check this by requiring the type of the value to be +drop-live. +The requirements for which are computed in `fn dropck_outlives`. + +The rest of this section uses the following type definition for a type +which requires its region parameter to be live: + +```rust +struct PrintOnDrop<'a>(&'a str); +impl<'a> Drop for PrintOnDrop<'_> { + fn drop(&mut self) { + println!("{}", self.0); + } +} +``` + +## How values are dropped + +At its core, a value of type `T` is dropped by executing its "drop +glue". Drop glue is compiler generated and first calls `::drop` and then recursively calls the drop glue of any recursively +owned values. + +- If `T` has an explicit `Drop` impl, call `::drop`. +- Regardless of whether `T` implements `Drop`, recurse into all values + *owned* by `T`: + - references, raw pointers, function pointers, function items, trait + objects[^traitobj], and scalars do not own anything. + - tuples, slices, and arrays consider their elements to be owned. + For arrays of length zero we do not own any value of the element + type. + - all fields (of all variants) of ADTs are considered owned. We + consider all variants for enums. The exception here is + `ManuallyDrop` which is not considered to own `U`. + `PhantomData` also does not own anything. + closures and generators own their captured upvars. + +Whether a type has drop glue is returned by [`fn +Ty::needs_drop`](https://github.com/rust-lang/rust/blob/320b412f9c55bf480d26276ff0ab480e4ecb29c0/compiler/rustc_middle/src/ty/util.rs#L1086-L1108). + +### Partially dropping a local + +For types which do not implement `Drop` themselves, we can also +partially move parts of the value before dropping the rest. In this case +only the drop glue for the not-yet moved values is called, e.g. + +```rust +fn main() { + let mut x = (PrintOnDrop("third"), PrintOnDrop("first")); + drop(x.1); + println!("second") +} +``` + +During MIR building we assume that a local may get dropped whenever it +goes out of scope *as long as its type needs drop*. Computing the exact +drop glue for a variable happens **after** borrowck in the +`ElaborateDrops` pass. This means that even if some part of the local +have been dropped previously, dropck still requires this value to be +live. This is the case even if we completely moved a local. + +```rust +fn main() { + let mut x; + { + let temp = String::from("I am temporary"); + x = PrintOnDrop(&temp); + drop(x); + } +} //~ ERROR `temp` does not live long enough. +``` + +It should be possible to add some amount of drop elaboration before +borrowck, allowing this example to compile. There is an unstable feature +to move drop elaboration before const checking: +[#73255](https://github.com/rust-lang/rust/issues/73255). Such a feature +gate does not exist for doing some drop elaboration before borrowck, +although there's a [relevant +MCP](https://github.com/rust-lang/compiler-team/issues/558). + +[^traitobj]: you can consider trait objects to have a builtin `Drop` +implementation which directly uses the `drop_in_place` provided by the +vtable. This `Drop` implementation requires all its generic parameters +to be live. + +### `dropck_outlives` + +There are two distinct "liveness" computations that we perform: + +* a value `v` is *use-live* at location `L` if it may be "used" later; a + *use* here is basically anything that is not a *drop* +* a value `v` is *drop-live* at location `L` if it maybe dropped later + +When things are *use-live*, their entire type must be valid at `L`. When +they are *drop-live*, all regions that are required by dropck must be +valid at `L`. The values dropped in the MIR are *places*. + +The constraints computed by `dropck_outlives` for a type closely match +the generated drop glue for that type. Unlike drop glue, +`dropck_outlives` cares about the types of owned values, not the values +itself. For a value of type `T` + +- if `T` has an explicit `Drop`, require all generic arguments to be + live, unless they are marked with `#[may_dangle]` in which case they + are fully ignored +- regardless of whether `T` has an explicit `Drop`, recurse into all + types *owned* by `T` + - references, raw pointers, function pointers, function items, trait + objects[^traitobj], and scalars do not own anything. + - tuples, slices and arrays consider their element type to be owned. + **For arrays we currently do not check whether their length is + zero**. + - all fields (of all variants) of ADTs are considered owned. The + exception here is `ManuallyDrop` which is not considered to own + `U`. **We consider `PhantomData` to own `U`**. + - closures and generators own their captured upvars. + +The sections marked in bold are cases where `dropck_outlives` considers +types to be owned which are ignored by `Ty::needs_drop`. We only rely on +`dropck_outlives` if `Ty::needs_drop` for the containing local returned +`true`.This means liveness requirements can change depending on whether +a type is contained in a larger local. **This is inconsistent, and +should be fixed: an example [for +arrays](https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=8b5f5f005a03971b22edb1c20c5e6cbe) +and [for +`PhantomData`](https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=44c6e2b1fae826329fd54c347603b6c8).**[^core] + +One possible way these inconsistencies can be fixed is by MIR building +to be more pessimistic, probably by making `Ty::needs_drop` weaker, or +alternatively, changing `dropck_outlives` to be more precise, requiring +fewer regions to be live. diff --git a/src/borrow_check/region_inference/lifetime_parameters.md b/src/borrow_check/region_inference/lifetime_parameters.md index 2dbbc0c1e..bd8a07843 100644 --- a/src/borrow_check/region_inference/lifetime_parameters.md +++ b/src/borrow_check/region_inference/lifetime_parameters.md @@ -43,8 +43,8 @@ only variant of [`ty::RegionKind`] that we use is the [`ReVar`] variant. These region variables are broken into two major categories, based on their index: -[`ty::RegionKind`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.RegionKind.html -[`ReVar`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.RegionKind.html#variant.ReVar +[`ty::RegionKind`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.RegionKind.html +[`ReVar`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.RegionKind.html#variant.ReVar - 0..N: universal regions -- the ones we are discussing here. In this case, the code must be correct with respect to any value of those diff --git a/src/early-late-bound.md b/src/bound-vars-and-params.md similarity index 54% rename from src/early-late-bound.md rename to src/bound-vars-and-params.md index 29b2136b4..50d5cefb1 100644 --- a/src/early-late-bound.md +++ b/src/bound-vars-and-params.md @@ -1,50 +1,4 @@ -# Early and Late Bound Variables - -In Rust, item definitions (like `fn`) can often have generic parameters, which -are always [_universally_ quantified][quant]. That is, if you have a function -like - -```rust -fn foo(x: T) { } -``` - -this function is defined "for all T" (not "for some specific T", which would be -[_existentially_ quantified][quant]). - -[quant]: ./appendix/background.md#quantified - -While Rust *items* can be quantified over types, lifetimes, and constants, the -types of values in Rust are only ever quantified over lifetimes. So you can -have a type like `for<'a> fn(&'a u32)`, which represents a function pointer -that takes a reference with any lifetime, or `for<'a> dyn Trait<'a>`, which is -a `dyn` trait for a trait implemented for any lifetime; but we have no type -like `for fn(T)`, which would be a function that takes a value of *any type* -as a parameter. This is a consequence of monomorphization -- to support a value -of type `for fn(T)`, we would need a single function pointer that can be -used for a parameter of any type, but in Rust we generate customized code for -each parameter type. - -One consequence of this asymmetry is a weird split in how we represent some -generic types: _early-_ and _late-_ bound parameters. -Basically, if we cannot represent a type (e.g. a universally quantified type), -we have to bind it _early_ so that the unrepresentable type is never around. - -Consider the following example: - -```rust,ignore -fn foo<'a, 'b, T>(x: &'a u32, y: &'b T) where T: 'b { ... } -``` - -We cannot treat `'a`, `'b`, and `T` in the same way. Types in Rust can't have -`for { .. }`, only `for<'a> {...}`, so whenever you reference `foo` the type -you get back can't be `for<'a, 'b, T> fn(&'a u32, y: &'b T)`. Instead, the `T` -must be substituted early. In particular, you have: - -```rust,ignore -let x = foo; // T, 'b have to be substituted here -x(...); // 'a substituted here, at the point of call -x(...); // 'a substituted here with a different value -``` +# Bound vars and parameters ## Early-bound parameters @@ -75,9 +29,7 @@ in [this chapter](./generics.md). ## Late-bound parameters -Late-bound parameters in `rustc` are handled quite differently (they are also -specialized to lifetimes since, right now, only late-bound lifetimes are -supported, though with GATs that has to change). We indicate their potential +Late-bound parameters in `rustc` are handled differently. We indicate their presence by a [`Binder`] type. The [`Binder`] doesn't know how many variables there are at that binding level. This can only be determined by walking the type itself and collecting them. So a type like `for<'a, 'b> ('a, 'b)` would be diff --git a/src/bug-fix-procedure.md b/src/bug-fix-procedure.md index 2f5e24716..388e39b93 100644 --- a/src/bug-fix-procedure.md +++ b/src/bug-fix-procedure.md @@ -1,4 +1,4 @@ -# Rustc Bug Fix Procedure +# Procedures for Breaking Changes @@ -169,6 +169,13 @@ there were no errors before. #### Crater and crates.io +[Crater] is a bot that will compile all crates.io crates and many +public github repos with the compiler with your changes. A report will then be +generated with crates that ceased to compile with or began to compile with your +changes. Crater runs can take a few days to complete. + +[Crater]: ./tests/crater.md + We should always do a crater run to assess impact. It is polite and considerate to at least notify the authors of affected crates the breaking change. If we can submit PRs to fix the problem, so much the better. @@ -297,7 +304,7 @@ self.tcx.sess.add_lint(lint::builtin::OVERLAPPING_INHERENT_IMPLS, We want to convert this into an error. In some cases, there may be an existing error for this scenario. In others, we will need to allocate a fresh diagnostic code. [Instructions for allocating a fresh diagnostic -code can be found here.](./diagnostics/diagnostic-codes.md) You may want +code can be found here.](./diagnostics/error-codes.md) You may want to mention in the extended description that the compiler behavior changed on this point, and include a reference to the tracking issue for the change. @@ -318,7 +325,7 @@ general, if the test used to have `#[deny(overlapping_inherent_impls)]`, that can just be removed. ``` -./x.py test +./x test ``` #### All done! diff --git a/src/building/bootstrapping.md b/src/building/bootstrapping.md index 3bacc21d3..936c75f36 100644 --- a/src/building/bootstrapping.md +++ b/src/building/bootstrapping.md @@ -1,8 +1,7 @@ -# Bootstrapping the Compiler +# Bootstrapping the compiler - [*Bootstrapping*][boot] is the process of using a compiler to compile itself. More accurately, it means using an older compiler to compile a newer version of the same compiler. @@ -16,8 +15,20 @@ version. This is exactly how `x.py` works: it downloads the current beta release of rustc, then uses it to compile the new compiler. +Note that this documentation mostly covers user-facing information. See +[bootstrap/README.md][bootstrap-internals] to read about bootstrap internals. + +[bootstrap-internals]: https://github.com/rust-lang/rust/blob/master/src/bootstrap/README.md + ## Stages of bootstrapping +### Overview + +- Stage 0: the pre-compiled compiler +- Stage 1: from current code, by an earlier compiler +- Stage 2: the truly current compiler +- Stage 3: the same-result test + Compiling `rustc` is done in stages. Here's a diagram, adapted from Joshua Nelson's [talk on bootstrapping][rustconf22-talk] at RustConf 2022, with detailed explanations below. @@ -28,7 +39,7 @@ stage0 compiler, and green nodes are built with the stage1 compiler. -[rustconf22-talk]: https://rustconf.com/schedule#bootstrapping-the-once-and-future-compiler +[rustconf22-talk]: https://www.youtube.com/watch?v=oUIjG-y4zaA ```mermaid graph TD @@ -47,7 +58,7 @@ graph TD classDef with-s1c fill: lightgreen; ``` -### Stage 0 +### Stage 0: the pre-compiled compiler The stage0 compiler is usually the current _beta_ `rustc` compiler and its associated dynamic libraries, @@ -61,11 +72,11 @@ a compiler (with its set of dependencies) and its 'target' or 'object' libraries (`std` and `rustc`). Both are staged, but in a staggered manner. -### Stage 1 +### Stage 1: from current code, by an earlier compiler The rustc source code is then compiled with the stage0 compiler to produce the stage1 compiler. -### Stage 2 +### Stage 2: the truly current compiler We then rebuild our stage1 compiler with itself to produce the stage2 compiler. @@ -85,10 +96,10 @@ However, it takes a very long time to build because one must first build the new compiler with an older compiler and then use that to build the new compiler with itself. For development, you usually only want the `stage1` compiler, -which you can build with `./x.py build library`. -See [Building the Compiler](./how-to-build-and-run.html#building-the-compiler). +which you can build with `./x build library`. +See [Building the compiler](./how-to-build-and-run.html#building-the-compiler). -### Stage 3 +### Stage 3: the same-result test Stage 3 is optional. To sanity check our new compiler, we can build the libraries with the stage2 compiler. The result ought @@ -96,7 +107,7 @@ to be identical to before, unless something has broken. ### Building the stages -`x.py` tries to be helpful and pick the stage you most likely meant for each subcommand. +`x` tries to be helpful and pick the stage you most likely meant for each subcommand. These defaults are as follows: - `check`: `--stage 0` @@ -135,38 +146,13 @@ bootstrapping the compiler. [intrinsics]: ../appendix/glossary.md#intrinsic [ocaml-compiler]: https://github.com/rust-lang/rust/tree/ef75860a0a72f79f97216f8aaa5b388d98da6480/src/boot -## Contributing to bootstrap - -When you use the bootstrap system, you'll call it through `x.py`. -However, most of the code lives in `src/bootstrap`. -`bootstrap` has a difficult problem: it is written in Rust, but yet it is run -before the Rust compiler is built! To work around this, there are two -components of bootstrap: the main one written in rust, and `bootstrap.py`. -`bootstrap.py` is what gets run by `x.py`. It takes care of downloading the -`stage0` compiler, which will then build the bootstrap binary written in -Rust. - -Because there are two separate codebases behind `x.py`, they need to -be kept in sync. In particular, both `bootstrap.py` and the bootstrap binary -parse `config.toml` and read the same command line arguments. `bootstrap.py` -keeps these in sync by setting various environment variables, and the -programs sometimes have to add arguments that are explicitly ignored, to be -read by the other. - -### Adding a setting to config.toml - -This section is a work in progress. In the meantime, you can see an example -contribution [here][bootstrap-build]. - -[bootstrap-build]: https://github.com/rust-lang/rust/pull/71994 - ## Understanding stages of bootstrap ### Overview This is a detailed look into the separate bootstrap stages. -The convention `x.py` uses is that: +The convention `x` uses is that: - A `--stage N` flag means to run the stage N compiler (`stageN/rustc`). - A "stage N artifact" is a build artifact that is _produced_ by the stage N compiler. @@ -175,7 +161,7 @@ The convention `x.py` uses is that: #### Build artifacts -Anything you can build with `x.py` is a _build artifact_. +Anything you can build with `x` is a _build artifact_. Build artifacts include, but are not limited to: - binaries, like `stage0-rustc/rustc-main` @@ -187,27 +173,27 @@ Build artifacts include, but are not limited to: #### Examples -- `./x.py build --stage 0` means to build with the beta `rustc`. -- `./x.py doc --stage 0` means to document using the beta `rustdoc`. -- `./x.py test --stage 0 library/std` means to run tests on the standard library - without building `rustc` from source ('build with stage 0, then test the +- `./x build --stage 0` means to build with the beta `rustc`. +- `./x doc --stage 0` means to document using the beta `rustdoc`. +- `./x test --stage 0 library/std` means to run tests on the standard library + without building `rustc` from source ('build with stage 0, then test the artifacts'). If you're working on the standard library, this is normally the test command you want. -- `./x.py test src/test/ui` means to build the stage 1 compiler and run +- `./x test tests/ui` means to build the stage 1 compiler and run `compiletest` on it. If you're working on the compiler, this is normally the test command you want. #### Examples of what *not* to do -- `./x.py test --stage 0 src/test/ui` is not useful: it runs tests on the - _beta_ compiler and doesn't build `rustc` from source. Use `test src/test/ui` +- `./x test --stage 0 tests/ui` is not useful: it runs tests on the + _beta_ compiler and doesn't build `rustc` from source. Use `test tests/ui` instead, which builds stage 1 from source. -- `./x.py test --stage 0 compiler/rustc` builds the compiler but runs no tests: +- `./x test --stage 0 compiler/rustc` builds the compiler but runs no tests: it's running `cargo test -p rustc`, but cargo doesn't understand Rust's tests. You shouldn't need to use this, use `test` instead (without arguments). -- `./x.py build --stage 0 compiler/rustc` builds the compiler, but does not build - libstd or even libcore. Most of the time, you'll want `./x.py build - library` instead, which allows compiling programs without needing to define +- `./x build --stage 0 compiler/rustc` builds the compiler, but does not build + libstd or even libcore. Most of the time, you'll want `./x build +library` instead, which allows compiling programs without needing to define lang items. ### Building vs. running @@ -257,7 +243,7 @@ artifacts into the appropriate place, skipping the cargo invocation. For instance, you might want to build an ARM version of rustc using an x86 machine. Building stage2 `std` is different when you are cross-compiling. -This is because `x.py` uses a trick: if `HOST` and `TARGET` are the same, +This is because `x` uses a trick: if `HOST` and `TARGET` are the same, it will reuse stage1 `std` for stage2! This is sound because stage1 `std` was compiled with the stage1 compiler, i.e. a compiler using the source code you currently have checked out. So it should be identical (and therefore ABI-compatible) @@ -270,6 +256,10 @@ So the stage2 compiler has to recompile `std` for the target. ### Why does only libstd use `cfg(bootstrap)`? +NOTE: for docs on `cfg(bootstrap)` itself, see [Complications of Bootstrapping][complications]. + +[complications]: #complications-of-bootstrapping + The `rustc` generated by the stage0 compiler is linked to the freshly-built `std`, which means that for the most part only `std` needs to be cfg-gated, so that `rustc` can use features added to std immediately after their addition, @@ -279,7 +269,7 @@ Note this is different from any other Rust program: stage1 `rustc` is built by the _beta_ compiler, but using the _master_ version of libstd! The only time `rustc` uses `cfg(bootstrap)` is when it adds internal lints -that use diagnostic items. This happens very rarely. +that use diagnostic items, or when it uses unstable library features that were recently changed. ### What is a 'sysroot'? @@ -372,7 +362,7 @@ You can find more discussion about sysroots in: ## Passing flags to commands invoked by `bootstrap` -`x.py` allows you to pass stage-specific flags to `rustc` and `cargo` when bootstrapping. +`x` allows you to pass stage-specific flags to `rustc` and `cargo` when bootstrapping. The `RUSTFLAGS_BOOTSTRAP` environment variable is passed as `RUSTFLAGS` to the bootstrap stage (stage0), and `RUSTFLAGS_NOT_BOOTSTRAP` is passed when building artifacts for later stages. `RUSTFLAGS` will work, but also affects the build of `bootstrap` itself, so it will be rare to want @@ -386,7 +376,7 @@ recompiling all dependencies. `CARGOFLAGS` will pass arguments to cargo itself (e.g. `--timings`). `CARGOFLAGS_BOOTSTRAP` and `CARGOFLAGS_NOT_BOOTSTRAP` work analogously to `RUSTFLAGS_BOOTSTRAP`. -`--test-args` will pass arguments through to the test runner. For `src/test/ui`, this is +`--test-args` will pass arguments through to the test runner. For `tests/ui`, this is compiletest; for unit tests and doctests this is the `libtest` runner. Most test runner accept `--help`, which you can use to find out the options accepted by the runner. @@ -405,46 +395,53 @@ If `./stageN/bin/rustc` gives an error about environment variables, that usually means something is quite wrong -- or you're trying to compile e.g. `rustc` or `std` or something that depends on environment variables. In the unlikely case that you actually need to invoke rustc in such a situation, -you can tell the bootstrap shim to print all env variables by adding `-vvv` to your `x.py` command. - -### Directories and artifacts generated by `bootstrap` - -This is an incomplete reference for the outputs generated by bootstrap: - -| Stage 0 Action | Output | -|-----------------------------------------------------------|----------------------------------------------| -| `beta` extracted | `build/HOST/stage0` | -| `stage0` builds `bootstrap` | `build/bootstrap` | -| `stage0` builds `test`/`std` | `build/HOST/stage0-std/TARGET` | -| copy `stage0-std` (HOST only) | `build/HOST/stage0-sysroot/lib/rustlib/HOST` | -| `stage0` builds `rustc` with `stage0-sysroot` | `build/HOST/stage0-rustc/HOST` | -| copy `stage0-rustc` (except executable) | `build/HOST/stage0-sysroot/lib/rustlib/HOST` | -| build `llvm` | `build/HOST/llvm` | -| `stage0` builds `codegen` with `stage0-sysroot` | `build/HOST/stage0-codegen/HOST` | -| `stage0` builds `rustdoc`, `clippy`, `miri`, with `stage0-sysroot` | `build/HOST/stage0-tools/HOST` | - -`--stage=0` stops here. - -| Stage 1 Action | Output | -|-----------------------------------------------------|---------------------------------------| -| copy (uplift) `stage0-rustc` executable to `stage1` | `build/HOST/stage1/bin` | -| copy (uplift) `stage0-codegen` to `stage1` | `build/HOST/stage1/lib` | -| copy (uplift) `stage0-sysroot` to `stage1` | `build/HOST/stage1/lib` | -| `stage1` builds `test`/`std` | `build/HOST/stage1-std/TARGET` | -| copy `stage1-std` (HOST only) | `build/HOST/stage1/lib/rustlib/HOST` | -| `stage1` builds `rustc` | `build/HOST/stage1-rustc/HOST` | -| copy `stage1-rustc` (except executable) | `build/HOST/stage1/lib/rustlib/HOST` | -| `stage1` builds `codegen` | `build/HOST/stage1-codegen/HOST` | - -`--stage=1` stops here. - -| Stage 2 Action | Output | -|--------------------------------------------------------|-----------------------------------------------------------------| -| copy (uplift) `stage1-rustc` executable | `build/HOST/stage2/bin` | -| copy (uplift) `stage1-sysroot` | `build/HOST/stage2/lib and build/HOST/stage2/lib/rustlib/HOST` | -| `stage2` builds `test`/`std` (not HOST targets) | `build/HOST/stage2-std/TARGET` | -| copy `stage2-std` (not HOST targets) | `build/HOST/stage2/lib/rustlib/TARGET` | -| `stage2` builds `rustdoc`, `clippy`, `miri` | `build/HOST/stage2-tools/HOST` | -| copy `rustdoc` | `build/HOST/stage2/bin` | - -`--stage=2` stops here. +you can tell the bootstrap shim to print all env variables by adding `-vvv` to your `x` command. + +Finally, bootstrap makes use of the [cc-rs crate] which has [its own +method][env-vars] of configuring C compilers and C flags via environment +variables. + +[cc-rs crate]: https://github.com/rust-lang/cc-rs +[env-vars]: https://github.com/rust-lang/cc-rs#external-configuration-via-environment-variables + +## Clarification of build command's stdout + +In this part, we will investigate the build command's stdout in an action +(similar, but more detailed and complete documentation compare to topic above). +When you execute `x build --dry-run` command, the build output will be something +like the following: + +```text +Building stage0 library artifacts (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu) +Copying stage0 library from stage0 (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu / x86_64-unknown-linux-gnu) +Building stage0 compiler artifacts (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu) +Copying stage0 rustc from stage0 (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu / x86_64-unknown-linux-gnu) +Assembling stage1 compiler (x86_64-unknown-linux-gnu) +Building stage1 library artifacts (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu) +Copying stage1 library from stage1 (x86_64-unknown-linux-gnu -> x86_64-unknown-linux-gnu / x86_64-unknown-linux-gnu) +Building stage1 tool rust-analyzer-proc-macro-srv (x86_64-unknown-linux-gnu) +Building rustdoc for stage1 (x86_64-unknown-linux-gnu) +``` + +### Building stage0 {std,compiler} artifacts + +These steps use the provided (downloaded, usually) compiler to compile the +local Rust source into libraries we can use. + +### Copying stage0 {std,rustc} + +This copies the library and compiler artifacts from Cargo into +`stage0-sysroot/lib/rustlib/{target-triple}/lib` + +### Assembling stage1 compiler + +This copies the libraries we built in "building stage0 ... artifacts" into +the stage1 compiler's lib directory. These are the host libraries that the +compiler itself uses to run. These aren't actually used by artifacts the new +compiler generates. This step also copies the rustc and rustdoc binaries we +generated into `build/$HOST/stage/bin`. + +The stage1/bin/rustc is a fully functional compiler, but it doesn't yet have +any libraries to link built binaries or libraries to. The next 3 steps will +provide those libraries for it; they are mostly equivalent to constructing +the stage1/bin compiler so we don't go through them individually. diff --git a/src/building/build-install-distribution-artifacts.md b/src/building/build-install-distribution-artifacts.md index 4ec3f958a..df3037469 100644 --- a/src/building/build-install-distribution-artifacts.md +++ b/src/building/build-install-distribution-artifacts.md @@ -3,22 +3,22 @@ You might want to build and package up the compiler for distribution. You’ll want to run this command to do it: - ```bash - ./x.py dist - ``` +```bash +./x dist +``` # Install distribution artifacts If you’ve built a distribution artifact you might want to install it and test that it works on your target system. You’ll want to run this command: - ```bash - ./x.py install - ``` +```bash +./x install +``` Note: If you are testing out a modification to a compiler, you might want to use it to compile some project. - Usually, you do not want to use `./x.py install` for testing. + Usually, you do not want to use `./x install` for testing. Rather, you should create a toolchain as discussed in [here][create-rustup-toolchain]. diff --git a/src/building/compiler-documenting.md b/src/building/compiler-documenting.md index 965e004c9..948571ce8 100644 --- a/src/building/compiler-documenting.md +++ b/src/building/compiler-documenting.md @@ -10,13 +10,13 @@ like the standard library (std) or the compiler (rustc). as rustdoc is under active development: ```bash - ./x.py doc + ./x doc ``` If you want to be sure the documentation looks the same as on CI: ```bash - ./x.py doc --stage 1 + ./x doc --stage 1 ``` This ensures that (current) rustdoc gets built, @@ -26,15 +26,17 @@ like the standard library (std) or the compiler (rustc). you can build just the documentation you want: ```bash - ./x.py doc src/doc/book - ./x.py doc src/doc/nomicon - ./x.py doc compiler library + ./x doc src/doc/book + ./x doc src/doc/nomicon + ./x doc compiler library ``` + See [the nightly docs index page](https://doc.rust-lang.org/nightly/) for a full list of books. + - Document internal rustc items Compiler documentation is not built by default. - To create it by default with `x.py doc`, modify `config.toml`: + To create it by default with `x doc`, modify `config.toml`: ```toml [build] diff --git a/src/building/how-to-build-and-run.md b/src/building/how-to-build-and-run.md index 46d4b9c04..1012e94fd 100644 --- a/src/building/how-to-build-and-run.md +++ b/src/building/how-to-build-and-run.md @@ -1,11 +1,10 @@ -# How to Build and Run the Compiler +# How to build and run the compiler + + The compiler is built using a tool called `x.py`. You will need to have Python installed to run it. -For instructions on how to install Python and other prerequisites, -see [the next page](./prerequisites.md). - ## Get the source code The main repository is [`rust-lang/rust`][repo]. This contains the compiler, @@ -21,69 +20,108 @@ git clone https://github.com/rust-lang/rust.git cd rust ``` -There are also submodules for things like LLVM, `clippy`, `miri`, etc. The -build tool will automatically clone and sync these for you. But if you want to, -you can do the following: +### Shallow clone the repository -```sh -# first time -git submodule update --init --recursive +Due to the size of the repository, cloning on a slower internet connection can take a long time. +To sidestep this, you can use the `--depth N` option with the `git clone` command. +This instructs `git` to perform a "shallow clone", cloning the repository but truncating it to +the last `N` commits. + +Passing `--depth 1` tells `git` to clone the repository but truncate the history to the latest +commit that is on the `master` branch, which is usually fine for browsing the source code or +building the compiler. -# subsequent times (to pull new commits) -git submodule update +```bash +git clone --depth 1 https://github.com/rust-lang/rust.git +cd rust ``` -## Create a `config.toml` +> **NOTE**: A shallow clone limits which `git` commands can be run. +> If you intend to work on and contribute to the compiler, it is +> generally recommended to fully clone the repository [as shown above](#get-the-source-code). +> +> For example, `git bisect` and `git blame` require access to the commit history, +> so they don't work if the repository was cloned with `--depth 1`. + +## What is `x.py`? -To start, run `./x.py setup`. This will do some initialization and create a -`config.toml` for you with reasonable defaults. These defaults are specified -indirectly via the `profile` setting, which points to one of the TOML files in -`src/bootstrap/defaults.` +`x.py` is the build tool for the `rust` repository. It can build docs, run tests, and compile the +compiler and standard library. -Alternatively, you can write `config.toml` by hand. See `config.toml.example` -for all the available settings and explanations of them. The following settings -are of particular interest, and `config.toml.example` has full explanations. +This chapter focuses on the basics to be productive, but +if you want to learn more about `x.py`, [read this chapter][bootstrap]. -You may want to change some of the following settings (and possibly others, such as -`llvm.ccache`): +[bootstrap]: ./bootstrapping.md -```toml -[llvm] -# Whether to use Rust CI built LLVM instead of locally building it. -download-ci-llvm = true # Download a pre-built LLVM? -assertions = true # LLVM assertions on? -ccache = "/path/to/ccache" # Use ccache when building LLVM? - -[rust] -debug-logging = true # Leave debug! and trace! calls in rustc? -incremental = true # Build rustc with incremental compilation? +Also, using `x` rather than `x.py` is recommended as: + +> `./x` is the most likely to work on every system (on Unix it runs the shell script +> that does python version detection, on Windows it will probably run the +> powershell script - certainly less likely to break than `./x.py` which often just +> opens the file in an editor).[^1] + +(You can find the platform related scripts around the `x.py`, like `x.ps1`) + +Notice that this is not absolute, for instance, using Nushell in VSCode on Win10, +typing `x` or `./x` still open the `x.py` in editor rather invoke the program :) + +In the rest of this guide, we use `x` rather than `x.py` directly. The following +command: + +```bash +./x check ``` -If you set `download-ci-llvm = true`, in some circumstances, such as when -updating the version of LLVM used by `rustc`, you may want to temporarily -disable this feature. See the ["Updating LLVM" section] for more. +could be replaced by: -["Updating LLVM" section]: ../backend/updating-llvm.md#feature-updates +```bash +./x.py check +``` -If you have already built `rustc` and you change settings related to LLVM, then you may have to -execute `rm -rf build` for subsequent configuration changes to take effect. Note that `./x.py -clean` will not cause a rebuild of LLVM. +### Running `x.py` -## What is `x.py`? +The `x.py` command can be run directly on most Unix systems in the following format: -`x.py` is the script used to orchestrate the tooling in the `rustc` repository. -It is the script that can build docs, run tests, and compile `rustc`. -It is the now preferred way to build `rustc` and it replaces the old makefiles -from before. Below are the different ways to utilize `x.py` in order to -effectively deal with the repo for various common tasks. +```sh +./x [flags] +``` -This chapter focuses on the basics to be productive, but -if you want to learn more about `x.py`, read its README.md -[here](https://github.com/rust-lang/rust/blob/master/src/bootstrap/README.md). -To read more about the bootstrap process and why `x.py` is necessary, -[read this chapter][bootstrap]. +This is how the documentation and examples assume you are running `x.py`. +Some alternative ways are: + +```sh +# On a Unix shell if you don't have the necessary `python3` command +./x [flags] -### Running `x.py` slightly more conveniently +# In Windows Powershell (if powershell is configured to run scripts) +./x [flags] +./x.ps1 [flags] + +# On the Windows Command Prompt (if .py files are configured to run Python) +x.py [flags] + +# You can also run Python yourself, e.g.: +python x.py [flags] +``` + +On Windows, the Powershell commands may give you an error that looks like this: +``` +PS C:\Users\vboxuser\rust> ./x +./x : File C:\Users\vboxuser\rust\x.ps1 cannot be loaded because running scripts is disabled on this system. For more +information, see about_Execution_Policies at https:/go.microsoft.com/fwlink/?LinkID=135170. +At line:1 char:1 ++ ./x ++ ~~~ + + CategoryInfo : SecurityError: (:) [], PSSecurityException + + FullyQualifiedErrorId : UnauthorizedAccess +``` + +You can avoid this error by allowing powershell to run local scripts: +``` +Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser +``` + +#### Running `x.py` slightly more conveniently There is a binary that wraps `x.py` called `x` in `src/tools/x`. All it does is run `x.py`, but it can be installed system-wide and run from any subdirectory @@ -91,51 +129,64 @@ of a checkout. It also looks up the appropriate version of `python` to use. You can install it with `cargo install --path src/tools/x`. -[bootstrap]: ./bootstrapping.md +To clarify that this is another global installed binary util, which is +similar to the one declared in section [What is `x.py`](#what-is-xpy), but +it works as an independent process to execute the `x.py` rather than calling the +shell to run the platform related scripts. -## Building the Compiler +## Create a `config.toml` -To build a compiler, run `./x.py build`. This will build up to the stage1 compiler, -including `rustdoc`, producing a usable compiler toolchain from the source -code you have checked out. +To start, run `./x setup` and select the `compiler` defaults. This will do some initialization +and create a `config.toml` for you with reasonable defaults. If you use a different default (which +you'll likely want to do if you want to contribute to an area of rust other than the compiler, such +as rustdoc), make sure to read information about that default (located in `src/bootstrap/defaults`) +as the build process may be different for other defaults. -Note that building will require a relatively large amount of storage space. -You may want to have upwards of 10 or 15 gigabytes available to build the compiler. +Alternatively, you can write `config.toml` by hand. See `config.example.toml` for all the available +settings and explanations of them. See `src/bootstrap/defaults` for common settings to change. -There are many flags you can pass to the build command of `x.py` that can be -beneficial to cutting down compile times or fitting other things you might -need to change. They are: - -```txt -Options: - -v, --verbose use verbose output (-vv for very verbose) - -i, --incremental use incremental compilation - --config FILE TOML configuration file for build - --build BUILD build target of the stage0 compiler - --host HOST host targets to build - --target TARGET target targets to build - --on-fail CMD command to run on failure - --stage N stage to build - --keep-stage N stage to keep without recompiling - --src DIR path to the root of the Rust checkout - -j, --jobs JOBS number of jobs to run in parallel - -h, --help print this help message -``` +If you have already built `rustc` and you change settings related to LLVM, then you may have to +execute `rm -rf build` for subsequent configuration changes to take effect. Note that `./x +clean` will not cause a rebuild of LLVM. -For hacking, often building the stage 1 compiler is enough, which saves a lot -of time. But for final testing and release, the stage 2 compiler is used. +## Common `x` commands -`./x.py check` is really fast to build the Rust compiler. -It is, in particular, very useful when you're doing some kind of -"type-based refactoring", like renaming a method, or changing the -signature of some function. +Here are the basic invocations of the `x` commands most commonly used when +working on `rustc`, `std`, `rustdoc`, and other tools. + +| Command | When to use it | +| ----------- | ------------------------------------------------------------------------------------------------------------ | +| `./x check` | Quick check to see if most things compile; [rust-analyzer can run this automatically for you][rust-analyzer] | +| `./x build` | Builds `rustc`, `std`, and `rustdoc` | +| `./x test` | Runs all tests | +| `./x fmt` | Formats all code | + +As written, these commands are reasonable starting points. However, there are +additional options and arguments for each of them that are worth learning for +serious development work. In particular, `./x build` and `./x test` +provide many ways to compile or test a subset of the code, which can save a lot +of time. + +Also, note that `x` supports all kinds of path suffixes for `compiler`, `library`, +and `src/tools` directories. So, you can simply run `x test tidy` instead of +`x test src/tools/tidy`. Or, `x build std` instead of `x build library/std`. + +[rust-analyzer]: suggested.html#configuring-rust-analyzer-for-rustc + +See the chapters on +[testing](../tests/running.md) and [rustdoc](../rustdoc.md) for more details. + +### Building the compiler + +Note that building will require a relatively large amount of storage space. +You may want to have upwards of 10 or 15 gigabytes available to build the compiler. Once you've created a `config.toml`, you are now ready to run -`x.py`. There are a lot of options here, but let's start with what is -probably the best "go to" command for building a local rust: +`x`. There are a lot of options here, but let's start with what is +probably the best "go to" command for building a local compiler: ```bash -./x.py build library +./x build library ``` This may *look* like it only builds the standard library, but that is not the case. @@ -156,8 +207,12 @@ see [the section on avoiding rebuilds for std][keep-stage]. [keep-stage]: ./suggested.md#faster-builds-with---keep-stage +Sometimes you don't need a full build. When doing some kind of +"type-based refactoring", like renaming a method, or changing the +signature of some function, you can use `./x check` instead for a much faster build. + Note that this whole command just gives you a subset of the full `rustc` -build. The **full** `rustc` build (what you get with `./x.py build +build. The **full** `rustc` build (what you get with `./x build --stage 2 compiler/rustc`) has quite a few more steps: - Build `rustc` with the stage1 compiler. @@ -167,16 +222,19 @@ build. The **full** `rustc` build (what you get with `./x.py build You almost never need to do this. -## Build specific components +### Build specific components If you are working on the standard library, you probably don't need to build the compiler unless you are planning to use a recently added nightly feature. Instead, you can just build using the bootstrap compiler. ```bash -./x.py build --stage 0 library +./x build --stage 0 library ``` +If you choose the `library` profile when running `x setup`, you can omit `--stage 0` (it's the +default). + ## Creating a rustup toolchain Once you have successfully built `rustc`, you will have created a bunch @@ -188,16 +246,11 @@ you will likely need to build at some point; for example, if you want to run the entire test suite). ```bash -rustup toolchain link stage1 build//stage1 -rustup toolchain link stage2 build//stage2 +rustup toolchain link stage0 build/host/stage0-sysroot # beta compiler + stage0 std +rustup toolchain link stage1 build/host/stage1 +rustup toolchain link stage2 build/host/stage2 ``` -The `` would typically be one of the following: - -- Linux: `x86_64-unknown-linux-gnu` -- Mac: `x86_64-apple-darwin` or `aarch64-apple-darwin` -- Windows: `x86_64-pc-windows-msvc` - Now you can run the `rustc` you built with. If you run with `-vV`, you should see a version number ending in `-dev`, indicating a build from your local environment: @@ -214,7 +267,7 @@ LLVM version: 11.0 ``` The rustup toolchain points to the specified toolchain compiled in your `build` directory, -so the rustup toolchain will be updated whenever `x.py build` or `x.py test` are run for +so the rustup toolchain will be updated whenever `x build` or `x test` are run for that toolchain/stage. **Note:** the toolchain we've built does not include `cargo`. In this case, `rustup` will @@ -229,25 +282,25 @@ custom toolchain for a project (e.g. via `rustup override set stage1`) you may want to build this component: ```bash -./x.py build proc-macro-srv-cli +./x build proc-macro-srv-cli ``` ## Building targets for cross-compilation To produce a compiler that can cross-compile for other targets, -pass any number of `target` flags to `x.py build`. +pass any number of `target` flags to `x build`. For example, if your host platform is `x86_64-unknown-linux-gnu` and your cross-compilation target is `wasm32-wasi`, you can build with: ```bash -./x.py build --target x86_64-unknown-linux-gnu --target wasm32-wasi +./x build --target x86_64-unknown-linux-gnu --target wasm32-wasi ``` Note that if you want the resulting compiler to be able to build crates that involve proc macros or build scripts, you must be sure to explicitly build target support for the host platform (in this case, `x86_64-unknown-linux-gnu`). -If you want to always build for other targets without needing to pass flags to `x.py build`, +If you want to always build for other targets without needing to pass flags to `x build`, you can configure this in the `[build]` section of your `config.toml` like so: ```toml @@ -259,7 +312,7 @@ Note that building for some targets requires having external dependencies instal (e.g. building musl targets requires a local copy of musl). Any target-specific configuration (e.g. the path to a local copy of musl) will need to be provided by your `config.toml`. -Please see `config.toml.example` for information on target-specific configuration keys. +Please see `config.example.toml` for information on target-specific configuration keys. For examples of the complete configuration necessary to build a target, please visit [the rustc book](https://doc.rust-lang.org/rustc/platform-support.html), @@ -276,24 +329,24 @@ then once you have built your compiler you will be able to use it to cross-compi cargo +stage1 build --target wasm32-wasi ``` -## Other `x.py` commands +## Other `x` commands -Here are a few other useful `x.py` commands. We'll cover some of them in detail +Here are a few other useful `x` commands. We'll cover some of them in detail in other sections: - Building things: - - `./x.py build` – builds everything using the stage 1 compiler, + - `./x build` – builds everything using the stage 1 compiler, not just up to `std` - - `./x.py build --stage 2` – builds everything with the stage 2 compiler including - `rustdoc` (which doesn't take too long) + - `./x build --stage 2` – builds everything with the stage 2 compiler including + `rustdoc` - Running tests (see the [section on running tests](../tests/running.html) for more details): - - `./x.py test library/std` – runs the unit tests and integration tests from `std` - - `./x.py test src/test/ui` – runs the `ui` test suite - - `./x.py test src/test/ui/const-generics` - runs all the tests in - the `const-generics/` subdirectory of the `ui` test suite - - `./x.py test src/test/ui/const-generics/const-types.rs` - runs - the single test `const-types.rs` from the `ui` test suite + - `./x test library/std` – runs the unit tests and integration tests from `std` + - `./x test tests/ui` – runs the `ui` test suite + - `./x test tests/ui/const-generics` - runs all the tests in + the `const-generics/` subdirectory of the `ui` test suite + - `./x test tests/ui/const-generics/const-types.rs` - runs + the single test `const-types.rs` from the `ui` test suite ### Cleaning out build directories @@ -303,8 +356,10 @@ you should file a bug as to what is going wrong. If you do need to clean everything up then you only need to run one command! ```bash -./x.py clean +./x clean ``` `rm -rf build` works too, but then you have to rebuild LLVM, which can take a long time even on fast computers. + +[^1]: issue[#1707](https://github.com/rust-lang/rustc-dev-guide/issues/1707) diff --git a/src/building/new-target.md b/src/building/new-target.md index f999a9472..83973f2c9 100644 --- a/src/building/new-target.md +++ b/src/building/new-target.md @@ -4,11 +4,13 @@ These are a set of steps to add support for a new target. There are numerous end states and paths to get there, so not all sections may be relevant to your desired goal. + + ## Specifying a new LLVM For very new targets, you may need to use a different fork of LLVM than what is currently shipped with Rust. In that case, navigate to -the `src/llvm-project` git submodule (you might need to run `./x.py +the `src/llvm-project` git submodule (you might need to run `./x check` at least once so the submodule is updated), check out the appropriate commit for your fork, then commit that new submodule reference in the main Rust repository. @@ -133,7 +135,7 @@ cross-compile `rustc`: ``` DESTDIR=/path/to/install/in \ -./x.py install -i --stage 1 --host aarch64-apple-darwin.json --target aarch64-apple-darwin \ +./x install -i --stage 1 --host aarch64-apple-darwin.json --target aarch64-apple-darwin \ compiler/rustc library/std ``` diff --git a/src/building/optimized-build.md b/src/building/optimized-build.md new file mode 100644 index 000000000..1e1136b91 --- /dev/null +++ b/src/building/optimized-build.md @@ -0,0 +1,129 @@ +# Optimized build of the compiler + + + +There are multiple additional build configuration options and techniques that can used to compile a +build of `rustc` that is as optimized as possible (for example when building `rustc` for a Linux +distribution). The status of these configuration options for various Rust targets is tracked [here]. +This page describes how you can use these approaches when building `rustc` yourself. + +[here]: https://github.com/rust-lang/rust/issues/103595 + +## Link-time optimization + +Link-time optimization is a powerful compiler technique that can increase program performance. To +enable (Thin-)LTO when building `rustc`, set the `rust.lto` config option to `"thin"` +in `config.toml`: + +```toml +[rust] +lto = "thin" +``` + +> Note that LTO for `rustc` is currently supported and tested only for +> the `x86_64-unknown-linux-gnu` target. Other targets *may* work, but no guarantees are provided. +> Notably, LTO-optimized `rustc` currently produces [miscompilations] on Windows. + +[miscompilations]: https://github.com/rust-lang/rust/issues/109114 + +Enabling LTO on Linux has [produced] speed-ups by up to 10%. + +[produced]: https://github.com/rust-lang/rust/pull/101403#issuecomment-1288190019 + +## Memory allocator + +Using a different memory allocator for `rustc` can provide significant performance benefits. If you +want to enable the `jemalloc` allocator, you can set the `rust.jemalloc` option to `true` +in `config.toml`: + +```toml +[rust] +jemalloc = true +``` + +> Note that this option is currently only supported for Linux and macOS targets. + +## Codegen units + +Reducing the amount of codegen units per `rustc` crate can produce a faster build of the compiler. +You can modify the number of codegen units for `rustc` and `libstd` in `config.toml` with the +following options: + +```toml +[rust] +codegen-units = 1 +codegen-units-std = 1 +``` + +## Instruction set + +By default, `rustc` is compiled for a generic (and conservative) instruction set architecture +(depending on the selected target), to make it support as many CPUs as possible. If you want to +compile `rustc` for a specific instruction set architecture, you can set the `target_cpu` compiler +option in `RUSTFLAGS`: + +```bash +RUSTFLAGS="-C target_cpu=x86-64-v3" ./x build ... +``` + +If you also want to compile LLVM for a specific instruction set, you can set `llvm` flags +in `config.toml`: + +```toml +[llvm] +cxxflags = "-march=x86-64-v3" +cflags = "-march=x86-64-v3" +``` + +## Profile-guided optimization + +Applying profile-guided optimizations (or more generally, feedback-directed optimizations) can +produce a large increase to `rustc` performance, by up to 15% ([1], [2]). However, these techniques +are not simply enabled by a configuration option, but rather they require a complex build workflow +that compiles `rustc` multiple times and profiles it on selected benchmarks. + +There is a tool called `opt-dist` that is used to optimize `rustc` with [PGO] (profile-guided +optimizations) and [BOLT] (a post-link binary optimizer) for builds distributed to end users. You +can examine the tool, which is located in `src/tools/opt-dist`, and build a custom PGO build +workflow based on it, or try to use it directly. Note that the tool is currently quite hardcoded to +the way we use it in Rust's continuous integration workflows, and it might require some custom +changes to make it work in a different environment. + +[1]: https://blog.rust-lang.org/inside-rust/2020/11/11/exploring-pgo-for-the-rust-compiler.html#final-numbers-and-a-benchmarking-plot-twist +[2]: https://github.com/rust-lang/rust/pull/96978 + +[PGO]: https://doc.rust-lang.org/rustc/profile-guided-optimization.html + +[BOLT]: https://github.com/llvm/llvm-project/blob/main/bolt/README.md + +To use the tool, you will need to provide some external dependencies: + +- A Python3 interpreter (for executing `x.py`). +- Compiled LLVM toolchain, with the `llvm-profdata` binary. Optionally, if you want to use BOLT, + the `llvm-bolt` and + `merge-fdata` binaries have to be available in the toolchain. + +These dependencies are provided to `opt-dist` by an implementation of the [`Environment`] struct. +It specifies directories where will the PGO/BOLT pipeline take place, and also external dependencies +like Python or LLVM. + +Here is an example of how can `opt-dist` be used locally (outside of CI): + +1. Build the tool with the following command: + ```bash + ./x build tools/opt-dist + ``` +2. Run the tool with the `local` mode and provide necessary parameters: + ```bash + ./build/host/stage0-tools-bin/opt-dist local \ + --target-triple \ # select target, e.g. "x86_64-unknown-linux-gnu" + --checkout-dir \ # path to rust checkout, e.g. "." + --llvm-dir \ # path to built LLVM toolchain, e.g. "/foo/bar/llvm/install" + -- python3 x.py dist # pass the actual build command + ``` + You can run `--help` to see further parameters that you can modify. + +[`Environment`]: https://github.com/rust-lang/rust/blob/ee451f8faccf3050c76cdcd82543c917b40c7962/src/tools/opt-dist/src/environment.rs#L5 + +> Note: if you want to run the actual CI pipeline, instead of running `opt-dist` locally, +> you can execute `DEPLOY=1 src/ci/docker/run.sh dist-x86_64-linux`. diff --git a/src/building/prerequisites.md b/src/building/prerequisites.md index 100b14aca..c511c7c81 100644 --- a/src/building/prerequisites.md +++ b/src/building/prerequisites.md @@ -2,48 +2,7 @@ ## Dependencies -Before building the compiler, you need the following things installed: - -* `python` 3 or 2.7 (under the name `python`; `python2` or `python3` will not work) -* `curl` -* `git` -* `ssl` which comes in `libssl-dev` or `openssl-devel` -* `pkg-config` if you are compiling on Linux and targeting Linux -* `libstdc++-static` may be required on some Linux distributions such as Fedora and Ubuntu - -If building LLVM from source (the default), you'll need additional tools: - -* `g++`, `clang++`, or MSVC with versions listed on - [LLVM's documentation](https://releases.llvm.org/13.0.0/docs/GettingStarted.html#host-c-toolchain-both-compiler-and-standard-library) -* `ninja`, or GNU `make` 3.81 or later (ninja is recommended, especially on Windows) -* `cmake` 3.13.4 or later - -Otherwise, you'll need LLVM installed and `llvm-config` in your path. -See [this section for more info][sysllvm]. - -[sysllvm]: ./new-target.md#using-pre-built-llvm - -### Windows - -* Install [winget](https://github.com/microsoft/winget-cli) - -`winget` is a Windows package manager. It will make package installation easy -on Windows. - -Run the following in a terminal: - -```powershell -winget install -e Python.Python.3 -winget install -e Kitware.CMake -``` - -If any of those is installed already, winget will detect it. Then edit your system's `PATH` variable -and add: `C:\Program Files\CMake\bin`. See -[this guide on editing the system `PATH`](https://www.java.com/en/download/help/path.html) from the -Java documentation. - -For more information about building on Windows, -see [the `rust-lang/rust` README](https://github.com/rust-lang/rust#building-on-windows). +See [the `rust-lang/rust` README](https://github.com/rust-lang/rust#dependencies). ## Hardware @@ -61,16 +20,16 @@ recommend trying to build on a Raspberry Pi! We recommend the following. * 2+ cores. Having more cores really helps. 10 or 20 or more is not too many! Beefier machines will lead to much faster builds. If your machine is not very -powerful, a common strategy is to only use `./x.py check` on your local machine +powerful, a common strategy is to only use `./x check` on your local machine and let the CI build test your changes when you push to a PR branch. Building the compiler takes more than half an hour on my moderately powerful -laptop. The first time you build the compiler, LLVM will also be built unless -you use CI-built LLVM ([see here][config]). +laptop. We suggest downloading LLVM from CI so you don't have to build it from source +([see here][config]). Like `cargo`, the build system will use as many cores as possible. Sometimes this can cause you to run low on memory. You can use `-j` to adjust the number -concurrent jobs. If a full build takes more than ~45 minutes to an hour, you +of concurrent jobs. If a full build takes more than ~45 minutes to an hour, you are probably spending most of the time swapping memory in and out; try using `-j1`. @@ -80,10 +39,3 @@ longer (especially after a rebase), but will save a ton of space from the incremental caches. [config]: ./how-to-build-and-run.md#create-a-configtoml - -## `rustc` and toolchain installation - -Follow the installation given in the [Rust book][install] to install a working -`rustc` and the necessary C/++ toolchain on your platform. - -[install]: https://doc.rust-lang.org/book/ch01-01-installation.html diff --git a/src/building/suggested.md b/src/building/suggested.md index a85229c6a..1336ff629 100644 --- a/src/building/suggested.md +++ b/src/building/suggested.md @@ -3,74 +3,40 @@ The full bootstrapping process takes quite a while. Here are some suggestions to make your life easier. + + ## Installing a pre-push hook CI will automatically fail your build if it doesn't pass `tidy`, our internal tool for ensuring code quality. If you'd like, you can install a [Git hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) -that will automatically run `./x.py test tidy` on each push, to ensure -your code is up to par. If the hook fails then run `./x.py test tidy --bless` +that will automatically run `./x test tidy` on each push, to ensure +your code is up to par. If the hook fails then run `./x test tidy --bless` and commit the changes. If you decide later that the pre-push behavior is undesirable, you can delete the `pre-push` file in `.git/hooks`. A prebuilt git hook lives at [`src/etc/pre-push.sh`](https://github.com/rust-lang/rust/blob/master/src/etc/pre-push.sh) which can be copied into your `.git/hooks` folder as `pre-push` (without the `.sh` extension!). -You can also install the hook as a step of running `./x.py setup`! +You can also install the hook as a step of running `./x setup`! ## Configuring `rust-analyzer` for `rustc` +### Visual Studio Code + `rust-analyzer` can help you check and format your code whenever you save a file. By default, `rust-analyzer` runs the `cargo check` and `rustfmt` commands, but you can override these commands to use more adapted versions -of these tools when hacking on `rustc`. For example, for Visual Studio Code, -you can write: - -```JSON -{ - "rust-analyzer.checkOnSave.overrideCommand": [ - "python3", - "x.py", - "check", - "--json-output" - ], - "rust-analyzer.rustfmt.overrideCommand": [ - "./build/$TARGET_TRIPLE/stage0/bin/rustfmt", - "--edition=2021" - ], - "rust-analyzer.procMacro.server": "./build/$TARGET_TRIPLE/stage0/libexec/rust-analyzer-proc-macro-srv", - "rust-analyzer.procMacro.enable": true, - "rust-analyzer.cargo.buildScripts.enable": true, - "rust-analyzer.cargo.buildScripts.invocationLocation": "root", - "rust-analyzer.cargo.buildScripts.invocationStrategy": "once", - "rust-analyzer.cargo.buildScripts.overrideCommand": [ - "python3", - "x.py", - "check", - "--json-output" - ], - "rust-analyzer.cargo.sysroot": "./build/$TARGET_TRIPLE/stage0-sysroot", - "rust-analyzer.rustc.source": "./Cargo.toml", -} -``` - -in your `.vscode/settings.json` file. This will ask `rust-analyzer` to use -`./x.py check` to check the sources, and the stage 0 rustfmt to format them. - -> NOTE: Make sure to replace `TARGET_TRIPLE` in the `rust-analyzer.rustfmt.overrideCommand` -> setting with the appropriate target triple for your machine. An example of such -> a triple is `x86_64-unknown-linux-gnu`. An easy way to check your target triple -> is to run `rustc -vV` and checking the `host` value of its output. +of these tools when hacking on `rustc`. For example, `x setup vscode` will prompt +you to create a `.vscode/settings.json` file which will configure Visual Studio code. +This will ask `rust-analyzer` to use `./x check` to check the sources, and the +stage 0 rustfmt to format them. +The recommended `rust-analyzer` settings live at [`src/etc/rust_analyzer_settings.json`]. -If you have enough free disk space and you would like to be able to run `x.py` commands while +If you have enough free disk space and you would like to be able to run `x` commands while rust-analyzer runs in the background, you can also add `--build-dir build-rust-analyzer` to the -`overrideCommand` to avoid x.py locking. +`overrideCommand` to avoid x locking. -If you're running `coc.nvim`, you can use `:CocLocalConfig` to create a -`.vim/coc-settings.json` and enter the same settings as above, but replacing -`editor.formatOnSave: true,` with -`"coc.preferences.formatOnSaveFiletypes": ["rust"],`. - -If running `./x.py check` on save is inconvenient, in VS Code you can use a [Build +If running `./x check` on save is inconvenient, in VS Code you can use a [Build Task] instead: ```JSON @@ -79,8 +45,8 @@ Task] instead: "version": "2.0.0", "tasks": [ { - "label": "./x.py check", - "command": "./x.py check", + "label": "./x check", + "command": "./x check", "type": "shell", "problemMatcher": "$rustc", "presentation": { "clear": true }, @@ -92,13 +58,42 @@ Task] instead: [Build Task]: https://code.visualstudio.com/docs/editor/tasks + +### Neovim + +For Neovim users there are several options for configuring for rustc. The easiest way is by using +[neoconf.nvim](https://github.com/folke/neoconf.nvim/), which allows for project-local +configuration files with the native LSP. The steps for how to use it are below. Note that requires +Rust-Analyzer to already be configured with Neovim. Steps for this can be +[found here](https://rust-analyzer.github.io/manual.html#nvim-lsp). + +1. First install the plugin. This can be done by following the steps in the README. +2. Run `x setup`, which will have a prompt for it to create a `.vscode/settings.json` file. + `neoconf` is able to read and update Rust-Analyzer settings automatically when the project is + opened when this file is detected. + +If you're running `coc.nvim`, +you can use `:CocLocalConfig` to create a `.vim/coc-settings.json`, +and copy the settings from [`src/etc/rust_analyzer_settings.json`]. + +Another way is without a plugin, and creating your own logic in your configuration. To do this you +must translate the JSON to Lua yourself. The translation is 1:1 and fairly straight-forward. It +must be put in the `["rust-analyzer"]` key of the setup table, which is +[shown here](https://github.com/neovim/nvim-lspconfig/blob/master/doc/server_configurations.md#rust_analyzer) + +If you would like to use the build task that is described above, you may either make your own +command in your config, or you can install a plugin such as +[overseer.nvim](https://github.com/stevearc/overseer.nvim) that can [read VSCode's `task.json` +files](https://github.com/stevearc/overseer.nvim/blob/master/doc/guides.md#vs-code-tasks), and +follow the same instructions as above. + ## Check, check, and check again -When doing simple refactorings, it can be useful to run `./x.py check` +When doing simple refactorings, it can be useful to run `./x check` continuously. If you set up `rust-analyzer` as described above, this will be done for you every time you save a file. Here you are just checking that the compiler can **build**, but often that is all you need (e.g., when renaming a -method). You can then run `./x.py build` when you actually need to +method). You can then run `./x build` when you actually need to run tests. In fact, it is sometimes useful to put off tests even when you are not @@ -109,6 +104,23 @@ the problem. A nice side-effect of this style is that you are left with a fairly fine-grained set of commits at the end, all of which build and pass tests. This often helps reviewing. +## `x suggest` + +The `x suggest` subcommand suggests (and runs) a subset of the extensive +`rust-lang/rust` tests based on files you have changed. This is especially useful +for new contributors who have not mastered the arcane `x` flags yet and more +experienced contributors as a shorthand for reducing mental effort. In all cases +it is useful not to run the full tests (which can take on the order of tens of +minutes) and just run a subset which are relevant to your changes. For example, +running `tidy` and `linkchecker` is useful when editing Markdown files, whereas UI +tests are much less likely to be helpful. While `x suggest` is a useful tool, it +does not guarantee perfect coverage (just as PR CI isn't a substitute for bors). +See the [dedicated chapter](../tests/suggest-tests.md) for more information and +contribution instructions. + +Please note that `x suggest` is in a beta state currently and the tests that it +will suggest are limited. + ## Configuring `rustup` to use nightly Some parts of the bootstrap process uses pinned, nightly versions of tools like @@ -124,11 +136,11 @@ directories you have [setup a worktree for]. You may need to use the pinned nightly version from `src/stage0.json`, but often the normal `nightly` channel will work. -**Note** see [the section on vscode] for how to configure it with this real rustfmt `x.py` uses, +**Note** see [the section on vscode] for how to configure it with this real rustfmt `x` uses, and [the section on rustup] for how to setup `rustup` toolchain for your bootstrapped compiler **Note** This does _not_ allow you to build `rustc` with cargo directly. You -still have to use `x.py` to work on the compiler or standard library, this just +still have to use `x` to work on the compiler or standard library, this just lets you use `cargo fmt`. [installing a nightly toolchain]: https://rust-lang.github.io/rustup/concepts/channels.html?highlight=nightl#working-with-nightly-rust @@ -141,7 +153,7 @@ lets you use `cargo fmt`. Sometimes just checking whether the compiler builds is not enough. A common example is that you need to add a `debug!` statement to inspect the value of some -state or better understand the problem. In that case, you really need +state or better understand the problem. In that case, you don't really need a full build. By bypassing bootstrap's cache invalidation, you can often get these builds to complete very fast (e.g., around 30 seconds). The only catch is this requires a bit of fudging and may produce compilers that @@ -149,13 +161,13 @@ don't work (but that is easily detected and fixed). The sequence of commands you want is as follows: -- Initial build: `./x.py build library` +- Initial build: `./x build library` - As [documented previously], this will build a functional stage1 compiler as part of running all stage0 commands (which include building a `std` compatible with the stage1 compiler) as well as the first few steps of the "stage 1 actions" up to "stage1 (sysroot stage1) builds std". -- Subsequent builds: `./x.py build library --keep-stage 1` +- Subsequent builds: `./x build library --keep-stage 1` - Note that we added the `--keep-stage 1` flag here [documented previously]: ./how-to-build-and-run.md#building-the-compiler @@ -177,8 +189,29 @@ rebuild. That ought to fix the problem. You can also use `--keep-stage 1` when running tests. Something like this: -- Initial test run: `./x.py test src/test/ui` -- Subsequent test run: `./x.py test src/test/ui --keep-stage 1` +- Initial test run: `./x test tests/ui` +- Subsequent test run: `./x test tests/ui --keep-stage 1` + +## Using incremental compilation + +You can further enable the `--incremental` flag to save additional +time in subsequent rebuilds: + +```bash +./x test tests/ui --incremental --test-args issue-1234 +``` + +If you don't want to include the flag with every command, you can +enable it in the `config.toml`: + +```toml +[rust] +incremental = true +``` + +Note that incremental compilation will use more disk space than usual. +If disk space is a concern for you, you might want to check the size +of the `build` directory from time to time. ## Fine-tuning optimizations @@ -240,13 +273,13 @@ If you're using nix, you can use the following nix-shell to work on Rust: # This file contains a development shell for working on rustc. let - # Build configuration for rust-lang/rust. Based on `config.toml.example` from - # `1bd30ce2aac40c7698aa4a1b9520aa649ff2d1c5`. + # Build configuration for rust-lang/rust. Based on `config.example.toml` (then called + # `config.toml.example`) from `1bd30ce2aac40c7698aa4a1b9520aa649ff2d1c5` config = pkgs.writeText "rustc-config" '' profile = "compiler" # you may want to choose a different profile, like `library` or `tools` - changelog-seen = 2 [build] + patch-binaries-for-nix = true # The path to (or name of) the GDB executable to use. This is only used for # executing the debuginfo test suite. gdb = "${pkgs.gdb}/bin/gdb" @@ -270,7 +303,7 @@ let # Files that are ignored by ripgrep when searching. ignoreFile = pkgs.writeText "rustc-rgignore" '' configure - config.toml.example + config.example.toml x.py LICENSE-MIT LICENSE-APACHE @@ -301,3 +334,15 @@ pkgs.mkShell { RUST_BOOTSTRAP_CONFIG = config; } ``` + +## Shell Completions + +If you use Bash, Fish or PowerShell, you can find automatically-generated shell completion scripts for `x.py` in [`src/etc/completions`](https://github.com/rust-lang/rust/tree/master/src/etc/completions). +Zsh support will also be included once issues with [`clap_complete`](https://crates.io/crates/clap_complete) have been resolved. + +You can use `source ./src/etc/completions/x.py.` +to load completions for your shell of choice, +or `source .\src\etc\completions\x.py.ps1` for PowerShell. +Adding this to your shell's startup script (e.g. `.bashrc`) will automatically load this completion. + +[`src/etc/rust_analyzer_settings.json`]: https://github.com/rust-lang/rust/blob/master/src/etc/rust_analyzer_settings.json diff --git a/src/closure.md b/src/closure.md index 5746fd4de..786c831ee 100644 --- a/src/closure.md +++ b/src/closure.md @@ -115,10 +115,10 @@ Let's start with defining a term that we will be using quite a bit in the rest o *upvar*. An **upvar** is a variable that is local to the function where the closure is defined. So, in the above examples, **x** will be an upvar to the closure. They are also sometimes referred to as the *free variables* meaning they are not bound to the context of the closure. -[`compiler/rustc_middle/src/ty/query/mod.rs`][upvars] defines a query called *upvars_mentioned* +[`compiler/rustc_passes/src/upvars.rs`][upvars] defines a query called *upvars_mentioned* for this purpose. -[upvars]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_query_impl/queries/struct.upvars_mentioned.html +[upvars]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_passes/upvars/index.html Other than lazy invocation, one other thing that distinguishes a closure from a normal function is that it can use the upvars. It borrows these upvars from its surrounding @@ -155,7 +155,7 @@ This uses the stage1 compiler and enables `debug!` logging for the The other option is to step through the code using lldb or gdb. -1. `rust-lldb build/x86_64-apple-darwin/stage1/bin/rustc test.rs` +1. `rust-lldb build/host/stage1/bin/rustc test.rs` 2. In lldb: 1. `b upvar.rs:134` // Setting the breakpoint on a certain line in the upvar.rs file` 2. `r` // Run the program until it hits the breakpoint diff --git a/src/compiler-debugging.md b/src/compiler-debugging.md index 35458b55c..65c3cadbb 100644 --- a/src/compiler-debugging.md +++ b/src/compiler-debugging.md @@ -1,5 +1,4 @@ # Debugging the compiler -[debugging]: #debugging @@ -43,7 +42,7 @@ otherwise you need to disable new symbol-mangling-version in `config.toml`. new-symbol-mangling = false ``` -> See the comments in `config.toml.example` for more info. +> See the comments in `config.example.toml` for more info. You will need to rebuild the compiler after changing any configuration option. @@ -128,7 +127,7 @@ fn main() { } ``` -```bash +``` $ rustc +stage1 error.rs error[E0277]: cannot add `()` to `{integer}` --> error.rs:2:7 @@ -143,7 +142,7 @@ error: aborting due to previous error Now, where does the error above come from? -```bash +``` $ RUST_BACKTRACE=1 rustc +stage1 error.rs -Z treat-err-as-bug error[E0277]: the trait bound `{integer}: std::ops::Add<()>` is not satisfied --> error.rs:2:7 @@ -185,6 +184,40 @@ stack backtrace: Cool, now I have a backtrace for the error! +## Getting the error creation location + +`-Z track-diagnostics` can help figure out where errors are emitted. It uses `#[track_caller]` +for this and prints its location alongside the error: + +``` +$ RUST_BACKTRACE=1 rustc +stage1 error.rs -Z track-diagnostics +error[E0277]: cannot add `()` to `{integer}` + --> src\error.rs:2:7 + | +2 | 1 + (); + | ^ no implementation for `{integer} + ()` +-Ztrack-diagnostics: created at compiler/rustc_trait_selection/src/traits/error_reporting/mod.rs:638:39 + | + = help: the trait `Add<()>` is not implemented for `{integer}` + = help: the following other types implement trait `Add`: + <&'a f32 as Add> + <&'a f64 as Add> + <&'a i128 as Add> + <&'a i16 as Add> + <&'a i32 as Add> + <&'a i64 as Add> + <&'a i8 as Add> + <&'a isize as Add> + and 48 others + +For more information about this error, try `rustc --explain E0277`. +``` + +This is similar but different to `-Z treat-err-as-bug`: +- it will print the locations for all errors emitted +- it does not require a compiler built with debug symbols +- you don't have to read through a big stack trace. + ## Getting logging output The compiler uses the [`tracing`] crate for logging. @@ -236,7 +269,7 @@ on *why* it was changed. See [this tutorial][bisect-tutorial] on how to use it. [bisect]: https://github.com/rust-lang/cargo-bisect-rustc -[bisect-tutorial]: https://github.com/rust-lang/cargo-bisect-rustc/blob/master/TUTORIAL.md +[bisect-tutorial]: https://rust-lang.github.io/cargo-bisect-rustc/tutorial.html ## Downloading Artifacts from Rust's CI @@ -307,3 +340,37 @@ error: aborting due to previous error ``` [`Layout`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_target/abi/struct.Layout.html + + +## Configuring CodeLLDB for debugging `rustc` + +If you are using VSCode, and have edited your `config.toml` to request debugging +level 1 or 2 for the parts of the code you're interested in, then you should be +able to use the [CodeLLDB] extension in VSCode to debug it. + +Here is a sample `launch.json` file, being used to run a stage 1 compiler direct +from the directory where it is built (does not have to be "installed"): + +```javascript +// .vscode/launch.json +{ + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Launch", + "args": [], // array of string command-line arguments to pass to compiler + "program": "${workspaceFolder}/build/host/stage1/bin/rustc", + "windows": { // applicable if using windows + "program": "${workspaceFolder}/build/host/stage1/bin/rustc.exe" + }, + "cwd": "${workspaceFolder}", // current working directory at program start + "stopOnEntry": false, + "sourceLanguages": ["rust"] + } + ] + } +``` + +[CodeLLDB]: https://marketplace.visualstudio.com/items?itemName=vadimcn.vscode-lldb diff --git a/src/compiler-src.md b/src/compiler-src.md index ad6968bb4..6090c6787 100644 --- a/src/compiler-src.md +++ b/src/compiler-src.md @@ -27,8 +27,10 @@ The repository consists of three main directories: `proc_macro`, `test`), as well as the Rust runtime (`backtrace`, `rtstartup`, `lang_start`). +- `tests/` contains the compiler tests. + - `src/` contains the source code for rustdoc, clippy, cargo, the build system, - compiler tests, language docs, etc. + language docs, etc. ## Compiler @@ -147,14 +149,14 @@ You can read more about rustdoc in [this chapter][rustdocch]. ## Tests -The test suite for all of the above is in [`src/test/`]. You can read more +The test suite for all of the above is in [`tests/`]. You can read more about the test suite [in this chapter][testsch]. The test harness itself is in [`src/tools/compiletest`]. [testsch]: ./tests/intro.md -[`src/test/`]: https://github.com/rust-lang/rust/tree/master/src/test +[`tests/`]: https://github.com/rust-lang/rust/tree/master/tests [`src/tools/compiletest`]: https://github.com/rust-lang/rust/tree/master/src/tools/compiletest ## Build System @@ -193,11 +195,8 @@ These include: run a lot of tests on a lot of platforms. - [`src/doc`]: Various documentation, including submodules for a few books. - [`src/etc`]: Miscellaneous utilities. -- [`src/tools/rustc-workspace-hack`], and others: Various workarounds to make - cargo work with bootstrapping. - And more... [`src/ci`]: https://github.com/rust-lang/rust/tree/master/src/ci [`src/doc`]: https://github.com/rust-lang/rust/tree/master/src/doc [`src/etc`]: https://github.com/rust-lang/rust/tree/master/src/etc -[`src/tools/rustc-workspace-hack`]: https://github.com/rust-lang/rust/tree/master/src/tools/rustc-workspace-hack diff --git a/src/compiler-team.md b/src/compiler-team.md index 4570fd3fa..d5ba78c77 100644 --- a/src/compiler-team.md +++ b/src/compiler-team.md @@ -124,7 +124,7 @@ for you. [reviewer rotation]: https://github.com/rust-lang/rust/blob/36285c5de8915ecc00d91ae0baa79a87ed5858d5/triagebot.toml#L528-L577 [triagebot]: https://github.com/rust-lang/triagebot/ -[automatically assigns]: https://github.com/rust-lang/triagebot/wiki/Assignment +[automatically assigns]: https://forge.rust-lang.org/triagebot/pr-assignment.html Getting on the reviewer rotation is much appreciated as it lowers the review burden for all of us! However, if you don't have time to give diff --git a/src/const-eval.md b/src/const-eval.md index a7b1c8963..736405326 100644 --- a/src/const-eval.md +++ b/src/const-eval.md @@ -40,52 +40,11 @@ in which the constant is evaluated (e.g. the function within which the constant and a [`GlobalId`]. The `GlobalId` is made up of an `Instance` referring to a constant or static or of an `Instance` of a function and an index into the function's `Promoted` table. -Constant evaluation returns an [`EvalToValTreeResult`] for type system constants or -[`EvalToConstValueResult`] with either the error, or a representation of the constant. - -Constants for the type system are encoded in "valtree representation". The `ValTree` datastructure -allows us to represent - -* arrays, -* many structs, -* tuples, -* enums and, -* most primitives. - -The basic rule for -being permitted in the type system is that every value must be uniquely represented. In other -words: a specific value must only be representable in one specific way. For example: there is only -one way to represent an array of two integers as a `ValTree`: -`ValTree::Branch(&[ValTree::Leaf(first_int), ValTree;:Leaf(second_int)])`. -Even though theoretically a `[u32; 2]` could be encoded in a `u64` and thus just be a -`ValTree::Leaf(bits_of_two_u32)`, that is not a legal construction of `ValTree` -(and is very complex to do, so it is unlikely anyone is tempted to do so). - -These rules also mean that some values are not representable. There can be no `union`s in type -level constants, as it is not clear how they should be represented, because their active variant -is unknown. Similarly there is no way to represent raw pointers, as addresses are unknown at -compile-time and thus we cannot make any assumptions about them. References on the other hand -*can* be represented, as equality for references is defined as equality on their value, so we -ignore their address and just look at the backing value. We must make sure that the pointer values -of the references are not observable at compile time. We thus encode `&42` exactly like `42`. -Any conversion from -valtree back to codegen constants must reintroduce an actual indirection. At codegen time the -addresses may be deduplicated between multiple uses or not, entirely depending on arbitrary -optimization choices. - -As a consequence, all decoding of `ValTree` must happen by matching on the type first and making -decisions depending on that. The value itself gives no useful information without the type that -belongs to it. - -Other constants get represented as [`ConstValue::Scalar`] or -[`ConstValue::Slice`] if possible. These values are only useful outside the -compile-time interpreter. If you need the value of a constant during -interpretation, you need to directly work with [`const_to_op`]. +Constant evaluation returns an [`EvalToValTreeResult`] for type system constants +or [`EvalToConstValueResult`] with either the error, or a representation of the +evaluated constant: a [valtree](mir/index.md#valtrees) or a [MIR constant +value](mir/index.md#mir-constant-values), respectively. [`GlobalId`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/struct.GlobalId.html -[`ConstValue::Scalar`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/value/enum.ConstValue.html#variant.Scalar -[`ConstValue::Slice`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/value/enum.ConstValue.html#variant.Slice -[`ConstValue::ByRef`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/value/enum.ConstValue.html#variant.ByRef [`EvalToConstValueResult`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/error/type.EvalToConstValueResult.html [`EvalToValTreeResult`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/interpret/error/type.EvalToValTreeResult.html -[`const_to_op`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_const_eval/interpret/struct.InterpCx.html#method.const_to_op diff --git a/src/const-eval/interpret.md b/src/const-eval/interpret.md index ee044505e..fbf781b96 100644 --- a/src/const-eval/interpret.md +++ b/src/const-eval/interpret.md @@ -82,7 +82,7 @@ The next statement asserts that said boolean is `0`. In case the assertion fails, its error message is used for reporting a compile-time error. Since it does not fail, `Operand::Immediate(Immediate::Scalar(Scalar::Raw { -data: 4054, .. }))` is stored in the virtual memory was allocated before the +data: 4054, .. }))` is stored in the virtual memory it was allocated before the evaluation. `_0` always refers to that location directly. After the evaluation is done, the return value is converted from [`Operand`] to diff --git a/src/constants.md b/src/constants.md index a33a283f3..56d01cc79 100644 --- a/src/constants.md +++ b/src/constants.md @@ -77,6 +77,6 @@ the constant doesn't use them in any way. This can cause [some interesting errors][pcg-unused-substs] and breaks some already stable code. [`ty::Const`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Const.html -[`ty::ConstKind`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.ConstKind.html -[`ty::TyKind`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.TyKind.html +[`ty::ConstKind`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.ConstKind.html +[`ty::TyKind`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.TyKind.html [pcg-unused-substs]: https://github.com/rust-lang/project-const-generics/issues/33 diff --git a/src/contributing.md b/src/contributing.md index 70017e5f5..d6b9fc84d 100644 --- a/src/contributing.md +++ b/src/contributing.md @@ -1,33 +1,8 @@ -# Contributing to Rust - -Thank you for your interest in contributing to Rust! There are many ways to -contribute, and we appreciate all of them. +# Contribution Procedures -If you have questions, please make a post on [internals.rust-lang.org][internals] or -hop on the [Rust Discord server][rust-discord] or [Rust Zulip server][rust-zulip]. - -As a reminder, all contributors are expected to follow our [Code of Conduct][coc]. - -If this is your first time contributing, the [Getting Started] and -[walkthrough] chapters can give you a good example of how a typical -contribution would go. - -[internals]: https://internals.rust-lang.org -[rust-discord]: http://discord.gg/rust-lang -[rust-zulip]: https://rust-lang.zulipchat.com -[coc]: https://www.rust-lang.org/conduct.html -[walkthrough]: ./walkthrough.md -[Getting Started]: ./getting-started.md - -## Feature Requests - -Feature requests need to go through a process to be approved by the relevant -teams. Usually this requires a Final Comment Period (FCP) or even a Request for -Comments (RFC). See [Getting Started] for more information about these processes. - -## Bug Reports +## Bug reports While bugs are unfortunate, they're a reality in software. We can't fix what we don't know about, so please report liberally. If you're not sure if something @@ -41,8 +16,7 @@ please follow our [instructions for reporting security vulnerabilities][vuln]**. If you're using the nightly channel, please check if the bug exists in the latest toolchain before filing your bug. It might be fixed already. -If you have the chance, before reporting a bug, please [search existing -issues](https://github.com/rust-lang/rust/issues?q=is%3Aissue), +If you have the chance, before reporting a bug, please [search existing issues], as it's possible that someone else has already reported your error. This doesn't always work, and sometimes it's hard to know what to search for, so consider this extra credit. We won't mind if you accidentally file a duplicate report. @@ -58,7 +32,79 @@ Opening an issue is as easy as following [this link](https://github.com/rust-lang/rust/issues/new/choose) and filling out the fields in the appropriate provided template. -## Pull Requests +## Bug fixes or "normal" code changes + +For most PRs, no special procedures are needed. You can just [open a PR], and it +will be reviewed, approved, and merged. This includes most bug fixes, +refactorings, and other user-invisible changes. The next few sections talk +about exceptions to this rule. + +Also, note that it is perfectly acceptable to open WIP PRs or GitHub [Draft PRs]. +Some people prefer to do this so they can get feedback along the +way or share their code with a collaborator. Others do this so they can utilize +the CI to build and test their PR (e.g. when developing on a slow machine). + +[open a PR]: #pull-requests +[Draft PRs]: https://github.blog/2019-02-14-introducing-draft-pull-requests/ + +## New features + +Rust has strong backwards-compatibility guarantees. Thus, new features can't +just be implemented directly in stable Rust. Instead, we have 3 release +channels: stable, beta, and nightly. + +- **Stable**: this is the latest stable release for general usage. +- **Beta**: this is the next release (will be stable within 6 weeks). +- **Nightly**: follows the `master` branch of the repo. This is the only + channel where unstable, incomplete, or experimental features are usable with + feature gates. + +See [this chapter on implementing new features](./implementing_new_features.md) for more +information. + +### Breaking changes + +Breaking changes have a [dedicated section][Breaking Changes] in the dev-guide. + +### Major changes + +The compiler team has a special process for large changes, whether or not they +cause breakage. This process is called a Major Change Proposal (MCP). MCP is a +relatively lightweight mechanism for getting feedback on large changes to the +compiler (as opposed to a full RFC or a design meeting with the team). + +Example of things that might require MCPs include major refactorings, changes +to important types, or important changes to how the compiler does something, or +smaller user-facing changes. + +**When in doubt, ask on [zulip]. It would be a shame to put a lot of work +into a PR that ends up not getting merged!** [See this document][mcpinfo] for +more info on MCPs. + +[mcpinfo]: https://forge.rust-lang.org/compiler/mcp.html +[zulip]: https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler + +### Performance + +Compiler performance is important. We have put a lot of effort over the last +few years into [gradually improving it][perfdash]. + +[perfdash]: https://perf.rust-lang.org/dashboard.html + +If you suspect that your change may cause a performance regression (or +improvement), you can request a "perf run" (and your reviewer may also request one +before approving). This is yet another bot that will compile a collection of +benchmarks on a compiler with your changes. The numbers are reported +[here][perf], and you can see a comparison of your changes against the latest +master. + +> For an introduction to the performance of Rust code in general +> which would also be useful in rustc development, see [The Rust Performance Book]. + +[perf]: https://perf.rust-lang.org +[The Rust Performance Book]: https://nnethercote.github.io/perf-book/ + +## Pull requests Pull requests (or PRs for short) are the primary mechanism we use to change Rust. GitHub itself has some [great documentation][about-pull-requests] on using the @@ -86,22 +132,59 @@ to the end of the pull request description, and [@rustbot] will assign them instead of a random person. This is entirely optional. You can also assign a random reviewer from a specific team by writing `r? rust-lang/groupname`. -So if you were making a diagnostics change, then you could get a reviewer from the diagnostics -team by adding: +As an example, +if you were making a diagnostics change, +then you could get a reviewer from the diagnostics team by adding: r? rust-lang/diagnostics -For a full list of possible `groupname` check the `adhoc_groups` section at the -[triagebot.toml config file](https://github.com/rust-lang/rust/blob/master/triagebot.toml) -or the list of teams in the [rust-lang teams -database](https://github.com/rust-lang/team/tree/master/teams). +For a full list of possible `groupname`s, +check the `adhoc_groups` section at the [triagebot.toml config file], +or the list of teams in the [rust-lang teams database]. + +### Waiting for reviews + +> NOTE +> +> Pull request reviewers are often working at capacity, +> and many of them are contributing on a volunteer basis. +> In order to minimize review delays, +> pull request authors and assigned reviewers should ensure that the review label +> (`S-waiting-on-review` and `S-waiting-on-author`) stays updated, +> invoking these commands when appropriate: +> +> - `@rustbot author`: +> the review is finished, +> and PR author should check the comments and take action accordingly. +> +> - `@rustbot review`: +> the author is ready for a review, +> and this PR will be queued again in the reviewer's queue. + +Please note that the reviewers are humans, who for the most part work on `rustc` +in their free time. This means that they can take some time to respond and review +your PR. It also means that reviewers can miss some PRs that are assigned to them. + +To try to move PRs forward, the Triage WG regularly goes through all PRs that +are waiting for review and haven't been discussed for at least 2 weeks. If you +don't get a review within 2 weeks, feel free to ask the Triage WG on +Zulip ([#t-release/triage]). They have knowledge of when to ping, who might be +on vacation, etc. + +The reviewer may request some changes using the GitHub code review interface. +They may also request special procedures for some PRs. +See [Crater] and [Breaking Changes] chapters for some examples of such procedures. + +[r?]: https://github.com/rust-lang/rust/pull/78133#issuecomment-712692371 +[#t-release/triage]: https://rust-lang.zulipchat.com/#narrow/stream/242269-t-release.2Ftriage +[Crater]: tests/crater.md ### CI -In addition to being reviewed by a human, pull requests are automatically tested +In addition to being reviewed by a human, pull requests are automatically tested, thanks to continuous integration (CI). Basically, every time you open and update a pull request, CI builds the compiler and tests it against the -[compiler test suite][rctd], and also performs other tests such as checking that +[compiler test suite], and also performs other tests such as checking that your pull request is in compliance with Rust's style guidelines. Running continuous integration tests allows PR authors to catch mistakes early @@ -111,7 +194,7 @@ of the status of a particular pull request. Rust has plenty of CI capacity, and you should never have to worry about wasting computational resources each time you push a change. It is also perfectly fine (and even encouraged!) to use the CI to test your changes if it can help your -productivity. In particular, we don't recommend running the full `./x.py test` suite locally, +productivity. In particular, we don't recommend running the full `./x test` suite locally, since it takes a very long time to execute. ### r+ @@ -122,7 +205,7 @@ on the pull request with an `r+`. It will look something like this: @bors r+ This tells [@bors], our lovable integration bot, that your pull request has -been approved. The PR then enters the [merge queue][merge-queue], where [@bors] +been approved. The PR then enters the [merge queue], where [@bors] will run *all* the tests on *every* platform we support. If it all works out, [@bors] will merge your code into `master` and close the pull request. @@ -135,27 +218,27 @@ Changes that are rolled up are tested and merged alongside other PRs, to speed the process up. Typically only small changes that are expected not to conflict with one another are marked as "always roll up". +Be patient; this can take a while and the queue can sometimes be long. PRs are never merged by hand. + [@rustbot]: https://github.com/rustbot [@bors]: https://github.com/bors -[merge-queue]: https://bors.rust-lang.org/queue/rust ### Opening a PR You are now ready to file a pull request? Great! Here are a few points you should be aware of. -All pull requests should be filed against the `master` branch, except in very -particular scenarios. Unless you know for sure that you should target another -branch, `master` will be the right choice (it's also the default). +All pull requests should be filed against the `master` branch, +unless you know for sure that you should target a different branch. Make sure your pull request is in compliance with Rust's style guidelines by running - $ ./x.py test tidy --bless + $ ./x test tidy --bless We recommend to make this check before every pull request (and every new commit -in a pull request); you can add [git hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks) -before every push to make sure you never forget to make this check. The -CI will also run tidy and will fail if tidy fails. +in a pull request); you can add [git hooks] +before every push to make sure you never forget to make this check. +The CI will also run tidy and will fail if tidy fails. Rust follows a _no merge-commit policy_, meaning, when you encounter merge conflicts you are expected to always rebase instead of merging. E.g. always use @@ -179,178 +262,29 @@ the issue in question. [labeling]: ./rustbot.md#issue-relabeling [closing-keywords]: https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue -### External Dependencies (subtree) - -As a developer to this repository, you don't have to treat the following external projects -differently from other crates that are directly in this repo: +## External dependencies -* [Clippy](https://github.com/rust-lang/rust-clippy) -* [Miri] -* [rustfmt](https://github.com/rust-lang/rustfmt) -* [rust-analyzer](https://github.com/rust-lang/rust-analyzer) +This section has moved to ["Using External Repositories"](./external-repos.md). -In contrast to `submodule` dependencies -(see below for those), the `subtree` dependencies are just regular files and directories which can -be updated in tree. However, if possible, enhancements, bug fixes, etc. specific -to these tools should be filed against the tools directly in their respective -upstream repositories. The exception is that when rustc changes are required to -implement a new tool feature or test, that should happen in one collective rustc PR. - -#### Synchronizing a subtree - -Periodically the changes made to subtree based dependencies need to be synchronized between this -repository and the upstream tool repositories. - -Subtree synchronizations are typically handled by the respective tool maintainers. Other users -are welcome to submit synchronization PRs, however, in order to do so you you will need to modify -your local git installation and follow a very precise set of instructions. -These instructions are documented, along with several useful tips and tricks, in the -[syncing subtree changes][clippy-sync-docs] section in Clippy's Contributing guide. -The instructions are applicable for use with any subtree based tool, just be sure to -use the correct corresponding subtree directory and remote repository. - -The synchronization process goes in two directions: `subtree push` and `subtree pull`. - -A `subtree push` takes all the changes that happened to the copy in this repo and creates commits -on the remote repo that match the local changes. Every local -commit that touched the subtree causes a commit on the remote repo, but -is modified to move the files from the specified directory to the tool repo root. - -A `subtree pull` takes all changes since the last `subtree pull` -from the tool repo and adds these commits to the rustc repo along with a merge commit that moves -the tool changes into the specified directory in the Rust repository. - -It is recommended that you always do a push first and get that merged to the tool master branch. -Then, when you do a pull, the merge works without conflicts. -While it's definitely possible to resolve conflicts during a pull, you may have to redo the conflict -resolution if your PR doesn't get merged fast enough and there are new conflicts. Do not try to -rebase the result of a `git subtree pull`, rebasing merge commits is a bad idea in general. - -You always need to specify the `-P` prefix to the subtree directory and the corresponding remote -repository. If you specify the wrong directory or repository -you'll get very fun merges that try to push the wrong directory to the wrong remote repository. -Luckily you can just abort this without any consequences by throwing away either the pulled commits -in rustc or the pushed branch on the remote and try again. It is usually fairly obvious -that this is happening because you suddenly get thousands of commits that want to be synchronized. - -[clippy-sync-docs]: https://doc.rust-lang.org/nightly/clippy/development/infrastructure/sync.html - -#### Creating a new subtree dependency - -If you want to create a new subtree dependency from an existing repository, call (from this -repository's root directory!) - -``` -git subtree add -P src/tools/clippy https://github.com/rust-lang/rust-clippy.git master -``` - -This will create a new commit, which you may not rebase under any circumstances! Delete the commit -and redo the operation if you need to rebase. - -Now you're done, the `src/tools/clippy` directory behaves as if Clippy were -part of the rustc monorepo, so no one but you (or others that synchronize -subtrees) actually needs to use `git subtree`. - - -### External Dependencies (submodules) - -Building Rust will also use external git repositories tracked using [git -submodules]. The complete list may be found in the [`.gitmodules`] file. Some -of these projects are required (like `stdarch` for the standard library) and -some of them are optional (like [Miri]). - -Usage of submodules is discussed more in the [Using Git -chapter](git.md#git-submodules). - -Some of the submodules are allowed to be in a "broken" state where they -either don't build or their tests don't pass, e.g. the documentation books -like [The Rust Reference]. Maintainers of these projects will be notified -when the project is in a broken state, and they should fix them as soon -as possible. The current status is tracked on the [toolstate website]. -More information may be found on the Forge [Toolstate chapter]. - -Breakage is not allowed in the beta and stable channels, and must be addressed -before the PR is merged. They are also not allowed to be broken on master in -the week leading up to the beta cut. - -[git submodules]: https://git-scm.com/book/en/v2/Git-Tools-Submodules -[`.gitmodules`]: https://github.com/rust-lang/rust/blob/master/.gitmodules -[The Rust Reference]: https://github.com/rust-lang/reference/ -[toolstate website]: https://rust-lang-nursery.github.io/rust-toolstate/ -[Toolstate chapter]: https://forge.rust-lang.org/infra/toolstate.html - -#### Breaking Tools Built With The Compiler - -Rust's build system builds a number of tools that make use of the internals of -the compiler and that are hosted in a separate repository, and included in Rust -via git submodules (such as [Miri]). If these tools break because of your -changes, you may run into a sort of "chicken and egg" problem. These tools rely -on the latest compiler to be built so you can't update them (in their own -repositories) to reflect your changes to the compiler until those changes are -merged into the compiler. At the same time, you can't get your changes merged -into the compiler because the rust-lang/rust build won't pass until those tools -build and pass their tests. - -Luckily, a feature was -[added to Rust's build](https://github.com/rust-lang/rust/issues/45861) to make -all of this easy to handle. The idea is that we allow these tools to be -"broken", so that the rust-lang/rust build passes without trying to build them, -then land the change in the compiler, and go update the tools that you -broke. Some tools will require waiting for a nightly release before this can -happen, while others use the builds uploaded after each bors merge and thus can -be updated immediately (check the tool's documentation for details). Once you're -done and the tools are working again, you go back in the compiler and update the -tools so they can be distributed again. - -This should avoid a bunch of synchronization dances and is also much easier on contributors as -there's no need to block on tools changes going upstream. - -Here are those same steps in detail: - -1. (optional) First, if it doesn't exist already, create a `config.toml` by copying - `config.toml.example` in the root directory of the Rust repository. - Set `submodules = false` in the `[build]` section. This will prevent `x.py` - from resetting to the original branch after you make your changes. If you - need to [update any submodules to their latest versions](#updating-submodules), - see the section of this file about that for more information. -2. (optional) Run `./x.py test src/tools/cargo` (substituting the submodule - that broke for `cargo`). Fix any errors in the submodule (and possibly others). -3. (optional) Make commits for your changes and send them to upstream repositories as a PR. -4. (optional) Maintainers of these submodules will **not** merge the PR. The PR can't be - merged because CI will be broken. You'll want to write a message on the PR referencing - your change, and how the PR should be merged once your change makes it into a nightly. -5. Wait for your PR to merge. -6. Wait for a nightly -7. (optional) Help land your PR on the upstream repository now that your changes are in nightly. -8. (optional) Send a PR to rust-lang/rust updating the submodule. - - -## Writing Documentation +## Writing documentation Documentation improvements are very welcome. The source of `doc.rust-lang.org` is located in [`src/doc`] in the tree, and standard API documentation is generated -from the source code itself (e.g. [`lib.rs`]). Documentation pull requests function -in the same way as other pull requests. +from the source code itself (e.g. [`library/std/src/lib.rs`][std-root]). Documentation pull requests +function in the same way as other pull requests. [`src/doc`]: https://github.com/rust-lang/rust/tree/master/src/doc -[`lib.rs`]: https://github.com/rust-lang/rust/blob/master/library/std/src/lib.rs#L1 - -To find documentation-related issues, sort by the [A-docs label][adocs]. - -[adocs]: https://github.com/rust-lang/rust/issues?q=is%3Aopen%20is%3Aissue%20label%3AA-docs +[std-root]: https://github.com/rust-lang/rust/blob/master/library/std/src/lib.rs#L1 -You can find documentation style guidelines in [RFC 1574][rfc1574]. +To find documentation-related issues, sort by the [A-docs label]. -[rfc1574]: https://github.com/rust-lang/rfcs/blob/master/text/1574-more-api-documentation-conventions.md#appendix-a-full-conventions-text +You can find documentation style guidelines in [RFC 1574]. -In many cases, you don't need a full `./x.py doc --stage 2`, which will build -the entire stage 2 compiler and compile the various books published on -[doc.rust-lang.org][docs]. When updating documentation for the standard library, -first try `./x.py doc library`. If that fails, or if you need to -see the output from the latest version of `rustdoc`, add `--stage 1`. -Results should appear in `build/$TARGET/doc`. - -[docs]: https://doc.rust-lang.org +To build the standard library documentation, use `x doc --stage 0 library --open`. +To build the documentation for a book (e.g. the unstable book), use `x doc src/doc/unstable-book.` +Results should appear in `build/host/doc`, as well as automatically open in your default browser. +See [Building Documentation](./building/compiler-documenting.md#building-documentation) for more +information. You can also use `rustdoc` directly to check small fixes. For example, `rustdoc src/doc/reference.md` will render reference to `doc/reference.html`. @@ -358,22 +292,23 @@ The CSS might be messed up, but you can verify that the HTML is right. ### Contributing to rustc-dev-guide -Contributions to the [rustc-dev-guide][rdg] are always welcome, and can be made directly at +Contributions to the [rustc-dev-guide] are always welcome, and can be made directly at [the rust-lang/rustc-dev-guide repo][rdgrepo]. The issue tracker in that repo is also a great way to find things that need doing. There are issues for beginners and advanced compiler devs alike! Just a few things to keep in mind: -- Please limit line length to 100 characters. This is enforced by CI, and you can run the checks - locally with `ci/check_line_lengths.sh`. +- Please limit line length to 100 characters. + This is enforced by CI, + and you can run the checks locally with `ci/lengthcheck.sh`. - When contributing text to the guide, please contextualize the information with some time period and/or a reason so that the reader knows how much to trust or mistrust the information. Aim to provide a reasonable amount of context, possibly including but not limited to: - - A reason for why the data may be out of date other than "change", as change is a constant across - the project. + - A reason for why the data may be out of date other than "change", + as change is a constant across the project. - The date the comment was added, e.g. instead of writing _"Currently, ..."_ or _"As of now, ..."_, @@ -390,20 +325,20 @@ Just a few things to keep in mind: add a special annotation before specifying the date: ```md - Jul 2022 + Jan 2023 ``` Example: ```md - As of Jul 2022, the foo did the bar. + As of Jan 2023, the foo did the bar. ``` For cases where the date should not be part of the visible rendered output, use the following instead: ```md - + ``` - A link to a relevant WG, tracking issue, `rustc` rustdoc page, or similar, that may provide @@ -414,139 +349,123 @@ Just a few things to keep in mind: subsections) it might benefit from having a Table of Contents at the beginning, which you can auto-generate by including the `` marker. -[rdg]: https://rustc-dev-guide.rust-lang.org/ -[rdgrepo]: https://github.com/rust-lang/rustc-dev-guide - -## Issue Triage +## Issue triage -Sometimes, an issue will stay open, even though the bug has been fixed. And -sometimes, the original bug may go stale because something has changed in the -meantime. +Sometimes, an issue will stay open, even though the bug has been fixed. +And sometimes, the original bug may go stale because something has changed in the meantime. -It can be helpful to go through older bug reports and make sure that they are -still valid. Load up an older issue, double check that it's still true, and -leave a comment letting us know if it is or is not. The [least recently -updated sort][lru] is good for finding issues like this. +It can be helpful to go through older bug reports and make sure that they are still valid. +Load up an older issue, double check that it's still true, +and leave a comment letting us know if it is or is not. +The [least recently updated sort][lru] is good for finding issues like this. [Thanks to `@rustbot`][rustbot], anyone can help triage issues by adding appropriate labels to issues that haven't been triaged yet: -* Yellow, **A**-prefixed labels state which **area** of the project an issue - relates to. - -* Magenta, **B**-prefixed labels identify bugs which are **blockers**. - -* Dark blue, **beta-** labels track changes which need to be backported into - the beta branches. - -* Light purple, **C**-prefixed labels represent the **category** of an issue. - -* Green, **E**-prefixed labels explain the level of **experience** necessary - to fix the issue. - -* The dark blue **final-comment-period** label marks bugs that are using the - RFC signoff functionality of [rfcbot] and are currently in the final - comment period. - -* Red, **I**-prefixed labels indicate the **importance** of the issue. The - [I-nominated][inom] label indicates that an issue has been nominated for - discussion at the next meeting of the team tagged using a - **T**-prefixed label. Similarly, the [I-prioritize][ipri] indicates - that an issue has been requested to be prioritized by the appropriate - team. - -* The purple **metabug** label marks lists of bugs collected by other - categories. - -* Purple gray, **O**-prefixed labels are the **operating system** or platform - that this issue is specific to. - -* Orange, **P**-prefixed labels indicate a bug's **priority**. These labels - can be assigned by anyone that understand the issue and is able to - prioritize it, and replace the [I-prioritize][ipri] label. - -* The gray **proposed-final-comment-period** label marks bugs that are using - the RFC signoff functionality of [rfcbot] and are currently awaiting - signoff of all team members in order to enter the final comment period. - -* Pink, **regression**-prefixed labels track regressions from stable to the - release channels. - -* The light orange **relnotes** label marks issues that should be documented in - the release notes of the next release. - -* Gray, **S**-prefixed labels are used for tracking the **status** of pull - requests. - -* Blue, **T**-prefixed bugs denote which **team** the issue belongs to. - -If you're looking for somewhere to start, check out the [E-easy][eeasy] tag. - -[rustbot]: ./rustbot.md -[inom]: https://github.com/rust-lang/rust/issues?q=is%3Aopen+is%3Aissue+label%3AI-nominated -[ipri]: https://github.com/rust-lang/rust/issues?q=is%3Aopen+is%3Aissue+label%3AI-prioritize -[eeasy]: https://github.com/rust-lang/rust/issues?q=is%3Aopen+is%3Aissue+label%3AE-easy [lru]: https://github.com/rust-lang/rust/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-asc -[rfcbot]: https://github.com/anp/rfcbot-rs/ +[rustbot]: ./rustbot.md -## Out-of-tree Contributions - -There are a number of other ways to contribute to Rust that don't deal with -rust-lang/rust: - -* Answer questions in the _Get Help!_ channels on the [Rust Discord - server][rust-discord], on [users.rust-lang.org][users], or on - [StackOverflow][so]. -* Participate in the [RFC process](https://github.com/rust-lang/rfcs). -* Find a [requested community library][community-library], build it, and publish - it to [Crates.io](http://crates.io). Easier said than done, but very, very - valuable! - -[rust-discord]: https://discord.gg/rust-lang -[users]: https://users.rust-lang.org/ -[so]: http://stackoverflow.com/questions/tagged/rust -[community-library]: https://github.com/rust-lang/rfcs/labels/A-community-library - -## Helpful Links and Information - -For people new to Rust, and just starting to contribute, or even for -more seasoned developers, some useful places to look for information -are: - -* This guide contains information about how various parts of the - compiler work and how to contribute to the compiler -* [Rust Forge][rustforge] contains additional documentation, including - write-ups of how to achieve common tasks -* The [Rust Internals forum][rif], a place to ask questions and - discuss Rust's internals -* The [generated documentation for Rust's compiler][gdfrustc] -* The [Rust reference][rr], even though it doesn't specifically talk about - Rust's internals, is a great resource nonetheless -* Although out of date, [Tom Lee's great blog article][tlgba] is very helpful -* [rustaceans.org][ro] is helpful, but mostly dedicated to IRC -* The [Rust Compiler Testing Docs][rctd] -* For [@bors], [this cheat sheet][cheatsheet] is helpful -* Google is always helpful when programming. - You can [search all Rust documentation][gsearchdocs] (the standard library, - the compiler, the books, the references, and the guides) to quickly find - information about the language and compiler. -* You can also use Rustdoc's built-in search feature to find documentation on - types and functions within the crates you're looking at. You can also search - by type signature! For example, searching for `* -> vec` should find all - functions that return a `Vec`. - _Hint:_ Find more tips and keyboard shortcuts by typing `?` on any Rustdoc - page! -* Don't be afraid to ask! The Rust community is friendly and helpful. - -[rustc dev guide]: about-this-guide.md -[gdfrustc]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ -[gsearchdocs]: https://www.google.com/search?q=site:doc.rust-lang.org+your+query+here -[stddocs]: https://doc.rust-lang.org/std -[rif]: http://internals.rust-lang.org -[rr]: https://doc.rust-lang.org/book/README.html -[rustforge]: https://forge.rust-lang.org/ -[tlgba]: https://tomlee.co/2014/04/a-more-detailed-tour-of-the-rust-compiler/ -[ro]: https://www.rustaceans.org/ -[rctd]: tests/intro.md -[cheatsheet]: https://bors.rust-lang.org/ -[Miri]: https://github.com/rust-lang/miri + + +| Labels | Color | Description | +|--------|-------|-------------| +| [A-] |   Yellow | The **area** of the project an issue relates to. | +| [B-] |   Magenta | Issues which are **blockers**. | +| [beta-] |   Dark Blue | Tracks changes which need to be [backported to beta][beta-backport] | +| [C-] |   Light Purple | The **category** of an issue. | +| [D-] |   Mossy Green | Issues for **diagnostics**. | +| [E-] |   Green | The **experience** level necessary to fix an issue. | +| [F-] |   Peach | Issues for **nightly features**. | +| [I-] |   Red | The **importance** of the issue. | +| [I-\*-nominated] |   Red | The issue has been nominated for discussion at the next meeting of the corresponding team. | +| [I-prioritize] |   Red | The issue has been nominated for prioritization by the team tagged with a **T**-prefixed label. | +| [metabug] |   Purple | Bugs that collect other bugs. | +| [O-] |   Purple Grey | The **operating system** or platform that the issue is specific to. | +| [P-] |   Orange | The issue **priority**. These labels can be assigned by anyone that understand the issue and is able to prioritize it, and remove the [I-prioritize] label. | +| [regression-] |   Pink | Tracks regressions from a stable release. | +| [relnotes] |   Light Orange | Changes that should be documented in the release notes of the next release. | +| [S-] |   Gray | Tracks the **status** of pull requests. | +| [S-tracking-] |   Steel Blue | Tracks the **status** of [tracking issues]. | +| [stable-] |   Dark Blue | Tracks changes which need to be [backported to stable][stable-backport] in anticipation of a point release. | +| [T-] |   Blue | Denotes which **team** the issue belongs to. | +| [WG-] |   Green | Denotes which **working group** the issue belongs to. | + + +[A-]: https://github.com/rust-lang/rust/labels?q=A +[B-]: https://github.com/rust-lang/rust/labels?q=B +[C-]: https://github.com/rust-lang/rust/labels?q=C +[D-]: https://github.com/rust-lang/rust/labels?q=D +[E-]: https://github.com/rust-lang/rust/labels?q=E +[F-]: https://github.com/rust-lang/rust/labels?q=F +[I-]: https://github.com/rust-lang/rust/labels?q=I +[O-]: https://github.com/rust-lang/rust/labels?q=O +[P-]: https://github.com/rust-lang/rust/labels?q=P +[S-]: https://github.com/rust-lang/rust/labels?q=S +[T-]: https://github.com/rust-lang/rust/labels?q=T +[WG-]: https://github.com/rust-lang/rust/labels?q=WG +[stable-]: https://github.com/rust-lang/rust/labels?q=stable +[beta-]: https://github.com/rust-lang/rust/labels?q=beta +[I-\*-nominated]: https://github.com/rust-lang/rust/labels?q=nominated +[I-prioritize]: https://github.com/rust-lang/rust/labels/I-prioritize +[tracking issues]: https://github.com/rust-lang/rust/labels/C-tracking-issue +[beta-backport]: https://forge.rust-lang.org/release/backporting.html#beta-backporting-in-rust-langrust +[stable-backport]: https://forge.rust-lang.org/release/backporting.html#stable-backporting-in-rust-langrust +[metabug]: https://github.com/rust-lang/rust/labels/metabug +[regression-]: https://github.com/rust-lang/rust/labels?q=regression +[relnotes]: https://github.com/rust-lang/rust/labels/relnotes +[S-tracking-]: https://github.com/rust-lang/rust/labels?q=s-tracking + +### Rfcbot labels + +[rfcbot] uses its own labels for tracking the process of coordinating +asynchronous decisions, such as approving or rejecting a change. +This is used for [RFCs], issues, and pull requests. + +| Labels | Color | Description | +|--------|-------|-------------| +| [proposed-final-comment-period] |   Gray | Currently awaiting signoff of all team members in order to enter the final comment period. | +| [disposition-merge] |   Green | Indicates the intent is to merge the change. | +| [disposition-close] |   Red | Indicates the intent is to not accept the change and close it. | +| [disposition-postpone] |   Gray | Indicates the intent is to not accept the change at this time and postpone it to a later date. | +| [final-comment-period] |   Blue | Currently soliciting final comments before merging or closing. | +| [finished-final-comment-period] |   Light Yellow | The final comment period has concluded, and the issue will be merged or closed. | +| [postponed] |   Yellow | The issue has been postponed. | +| [closed] |   Red | The issue has been rejected. | +| [to-announce] |   Gray | Issues that have finished their final-comment-period and should be publicly announced. Note: the rust-lang/rust repository uses this label differently, to announce issues at the triage meetings. | + +[disposition-merge]: https://github.com/rust-lang/rust/labels/disposition-merge +[disposition-close]: https://github.com/rust-lang/rust/labels/disposition-close +[disposition-postpone]: https://github.com/rust-lang/rust/labels/disposition-postpone +[proposed-final-comment-period]: https://github.com/rust-lang/rust/labels/proposed-final-comment-period +[final-comment-period]: https://github.com/rust-lang/rust/labels/final-comment-period +[finished-final-comment-period]: https://github.com/rust-lang/rust/labels/finished-final-comment-period +[postponed]: https://github.com/rust-lang/rfcs/labels/postponed +[closed]: https://github.com/rust-lang/rfcs/labels/closed +[to-announce]: https://github.com/rust-lang/rfcs/labels/to-announce +[rfcbot]: https://github.com/anp/rfcbot-rs/ +[RFCs]: https://github.com/rust-lang/rfcs + +## Helpful links and information + +This section has moved to the ["About this guide"] chapter. + +["About this guide"]: about-this-guide.md#other-places-to-find-information +[search existing issues]: https://github.com/rust-lang/rust/issues?q=is%3Aissue +[Breaking Changes]: bug-fix-procedure.md +[triagebot.toml config file]: https://github.com/rust-lang/rust/blob/HEAD/triagebot.toml +[rust-lang teams database]: https://github.com/rust-lang/team/tree/HEAD/teams +[compiler test suite]: tests/intro.md +[merge queue]: https://bors.rust-lang.org/queue/rust +[git hooks]: https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks +[A-docs label]: https://github.com/rust-lang/rust/issues?q=is%3Aopen%20is%3Aissue%20label%3AA-docs +[RFC 1574]: https://github.com/rust-lang/rfcs/blob/master/text/1574-more-api-documentation-conventions.md#appendix-a-full-conventions-text +[rustc-dev-guide]: https://rustc-dev-guide.rust-lang.org/ +[rdgrepo]: https://github.com/rust-lang/rustc-dev-guide diff --git a/src/conventions.md b/src/conventions.md index 4dd0a2da9..1c963c669 100644 --- a/src/conventions.md +++ b/src/conventions.md @@ -13,11 +13,11 @@ However, for now we don't use stable `rustfmt`; we use a pinned version with a special config, so this may result in different style from normal [`rustfmt`]. Therefore, formatting this repository using `cargo fmt` is not recommended. -Instead, formatting should be done using `./x.py fmt`. It's a good habit to run -`./x.py fmt` before every commit, as this reduces conflicts later. +Instead, formatting should be done using `./x fmt`. It's a good habit to run +`./x fmt` before every commit, as this reduces conflicts later. Formatting is checked by the `tidy` script. It runs automatically when you do -`./x.py test` and can be run in isolation with `./x.py fmt --check`. +`./x test` and can be run in isolation with `./x fmt --check`. If you want to use format-on-save in your editor, the pinned version of `rustfmt` is built under `build//stage0/bin/rustfmt`. You'll have to @@ -29,7 +29,10 @@ pass the `--edition=2021` argument yourself when c + + ### Copyright notice + In the past, files began with a copyright and license notice. Please **omit** this notice for new files licensed under the standard terms (dual @@ -123,7 +126,7 @@ dramatically (versus adding to it) in a later commit, that **Format liberally.** While only the final commit of a PR must be correctly formatted, it is both easier to review and less noisy to format each commit -individually using `./x.py fmt`. +individually using `./x fmt`. **No merges.** We do not allow merge commits into our history, other than those by bors. If you get a merge conflict, rebase instead via a diff --git a/src/crates-io.md b/src/crates-io.md index f012c5bb5..403d61a81 100644 --- a/src/crates-io.md +++ b/src/crates-io.md @@ -11,10 +11,9 @@ you should avoid adding dependencies to the compiler for several reasons: - The dependency may have transitive dependencies that have one of the above problems. -As of Aug 2022, -there is no official policy for vetting new dependencies to the compiler. -Decisions are made on a case-by-case basis, -during code review. + +Note that there is no official policy for vetting new dependencies to the compiler. +Decisions are made on a case-by-case basis, during code review. ## Permitted dependencies diff --git a/src/debugging-support-in-rustc.md b/src/debugging-support-in-rustc.md index 932b23b92..ac629934e 100644 --- a/src/debugging-support-in-rustc.md +++ b/src/debugging-support-in-rustc.md @@ -133,8 +133,7 @@ or `symbols`. Rust has support for embedding Natvis files for crates outside of the standard libraries by using the `#[debugger_visualizer]` attribute. For more details on how to embed debugger visualizers, -please refer to the `#[debugger_visualizer]` attribute in -[the unstable book](https://doc.rust-lang.org/unstable-book/language-features/debugger-visualizer.html). +please refer to the section on the [`debugger_visualizer` attribute]. ## DWARF and `rustc` @@ -352,3 +351,4 @@ but may have to add some mode to let the compiler understand some extensions. [symbol records]: https://llvm.org/docs/PDB/CodeViewSymbols.html [type records]: https://llvm.org/docs/PDB/CodeViewTypes.html [Windows Debugging Tools]: https://docs.microsoft.com/en-us/windows-hardware/drivers/debugger/ +[`debugger_visualizer` attribute]: https://doc.rust-lang.org/nightly/reference/attributes/debugger.html#the-debugger_visualizer-attribute diff --git a/src/diagnostics.md b/src/diagnostics.md index e1d5fbe1a..9f4245f28 100644 --- a/src/diagnostics.md +++ b/src/diagnostics.md @@ -69,17 +69,12 @@ surrounded with backticks: error: the identifier `foo.bar` is invalid ``` -### Error explanations +### Error codes and explanations -Some errors include long form descriptions. They may be viewed with the -`--explain` flag, or via the [error index]. Each explanation comes with an -example of how to trigger it and advice on how to fix it. - -Please read [RFC 1567] for details on how to format and write long error -codes. - -The descriptions are written in Markdown, and all of them are linked in the -[`rustc_error_codes`] crate. +Most errors have an associated error code. Error codes are linked to long-form +explanations which contains an example of how to trigger the error and in-depth +details about the error. They may be viewed with the `--explain` flag, or via +the [error index]. As a general rule, give an error a code (with an associated explanation) if the explanation would give more information than the error itself. A lot of the time @@ -89,12 +84,15 @@ triggers to include useful information for all cases in the error, in which case it's a good idea to add an explanation.[^estebank] As always, if you are not sure, just ask your reviewer! +If you decide to add a new error with an associated error code, please read +[this section][error-codes] for a guide and important details about the +process. + [^estebank]: This rule of thumb was suggested by **@estebank** [here][estebank-comment]. -[`rustc_error_codes`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_error_codes/error_codes/index.html [error index]: https://doc.rust-lang.org/error-index.html -[RFC 1567]: https://github.com/rust-lang/rfcs/blob/master/text/1567-long-error-codes-explanation-normalization.md [estebank-comment]: https://github.com/rust-lang/rustc-dev-guide/pull/967#issuecomment-733218283 +[error-codes]: ./diagnostics/error-codes.md ### Lints versus fixed diagnostics @@ -112,10 +110,11 @@ Here are a few examples: - Dead code: this is a lint. While the user probably doesn't want dead code in their crate, making this a hard error would make refactoring and development very painful. -- [safe_packed_borrows future compatibility warning][safe_packed_borrows]: - this is a silencable lint related to safety. It was judged that the making - this a hard (fixed) error would cause too much breakage, so instead a - warning is emitted that eventually will be turned into a hard error. +- [future-incompatible lints]: + these are silencable lints. + It was decided that making them fixed errors would cause too much breakage, + so warnings are instead emitted, + and will eventually be turned into fixed (hard) errors. Hard-coded warnings (those using the `span_warn` methods) should be avoided for normal code, preferring to use lints instead. Some cases, such as warnings @@ -124,7 +123,7 @@ with CLI flags, will require the use of hard-coded warnings. See the `deny` [lint level](#diagnostic-levels) below for guidelines when to use an error-level lint instead of a fixed error. -[safe_packed_borrows]: https://github.com/rust-lang/rust/issues/46043 +[future-incompatible lints]: #future-incompatible-lints ## Diagnostic output style guide @@ -269,7 +268,7 @@ book][rustc-lint-levels] and the [reference][reference-diagnostics]. ### Finding the source of errors -There are two main ways to find where a given error is emitted: +There are three main ways to find where a given error is emitted: - `grep` for either a sub-part of the error message/label or error code. This usually works well and is straightforward, but there are some cases where @@ -287,6 +286,8 @@ There are two main ways to find where a given error is emitted: - The _construction_ of the error is far away from where it is _emitted_, a problem similar to the one we faced with the `grep` approach. In some cases, we buffer multiple errors in order to emit them in order. +- Invoking `rustc` with `-Z track-diagnostics` will print error creation + locations alongside the error. The regular development practices apply: judicious use of `debug!()` statements and use of a debugger to trigger break points in order to figure out in what @@ -506,6 +507,83 @@ module. [rlint]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/lint/index.html +### When do lints run? + +Different lints will run at different times based on what information the lint +needs to do its job. Some lints get grouped into *passes* where the lints +within a pass are processed together via a single visitor. Some of the passes +are: + +- Pre-expansion pass: Works on [AST nodes] before [macro expansion]. This + should generally be avoided. + - Example: [`keyword_idents`] checks for identifiers that will become + keywords in future editions, but is sensitive to identifiers used in + macros. + +- Early lint pass: Works on [AST nodes] after [macro expansion] and name + resolution, just before [HIR lowering]. These lints are for purely + syntactical lints. + - Example: The [`unsued_parens`] lint checks for parenthesized-expressions + in situations where they are not needed, like an `if` condition. + +- Late lint pass: Works on [HIR nodes], towards the end of [analysis] (after + borrow checking, etc.). These lints have full type information available. + Most lints are late. + - Example: The [`invalid_value`] lint (which checks for obviously invalid + uninitialized values) is a late lint because it needs type information to + figure out whether a type allows being left uninitialized. + +- MIR pass: Works on [MIR nodes]. This isn't quite the same as other passes; + lints that work on MIR nodes have their own methods for running. + - Example: The [`arithmetic_overflow`] lint is emitted when it detects a + constant value that may overflow. + +Most lints work well via the pass systems, and they have a fairly +straightforward interface and easy way to integrate (mostly just implementing +a specific `check` function). However, some lints are easier to write when +they live on a specific code path anywhere in the compiler. For example, the +[`unused_mut`] lint is implemented in the borrow checker as it requires some +information and state in the borrow checker. + +Some of these inline lints fire before the linting system is ready. Those +lints will be *buffered* where they are held until later phases of the +compiler when the linting system is ready. See [Linting early in the +compiler](#linting-early-in-the-compiler). + + +[AST nodes]: the-parser.md +[HIR lowering]: lowering.md +[HIR nodes]: hir.md +[MIR nodes]: mir/index.md +[macro expansion]: macro-expansion.md +[analysis]: part-4-intro.md +[`keyword_idents`]: https://doc.rust-lang.org/rustc/lints/listing/allowed-by-default.html#keyword-idents +[`unsued_parens`]: https://doc.rust-lang.org/rustc/lints/listing/warn-by-default.html#unused-parens +[`invalid_value`]: https://doc.rust-lang.org/rustc/lints/listing/warn-by-default.html#invalid-value +[`arithmetic_overflow`]: https://doc.rust-lang.org/rustc/lints/listing/deny-by-default.html#arithmetic-overflow +[`unused_mut`]: https://doc.rust-lang.org/rustc/lints/listing/warn-by-default.html#unused-mut + +### Lint definition terms + +Lints are managed via the [`LintStore`][LintStore] and get registered in +various ways. The following terms refer to the different classes of lints +generally based on how they are registered. + +- *Built-in* lints are defined inside the compiler source. +- *Driver-registered* lints are registered when the compiler driver is created + by an external driver. This is the mechanism used by Clippy, for example. +- *Plugin* lints are registered by the [deprecated plugin system]. +- *Tool* lints are lints with a path prefix like `clippy::` or `rustdoc::`. +- *Internal* lints are the `rustc::` scoped tool lints that only run on the + rustc source tree itself and are defined in the compiler source like a + regular built-in lint. + +More information about lint registration can be found in the [LintStore] +chapter. + +[deprecated plugin system]: https://doc.rust-lang.org/nightly/unstable-book/language-features/plugin.html +[LintStore]: diagnostics/lintstore.md + ### Declaring a lint The built-in compiler lints are defined in the [`rustc_lint`][builtin] @@ -635,12 +713,12 @@ broader meaning than what rustc exposes to users of the compiler. Inside rustc, future-incompatible lints are for signalling to the user that code they have written may not compile in the future. In general, future-incompatible code exists for two reasons: -* the user has written unsound code that the compiler mistakenly accepted. While +* The user has written unsound code that the compiler mistakenly accepted. While it is within Rust's backwards compatibility guarantees to fix the soundness hole (breaking the user's code), the lint is there to warn the user that this will happen in some upcoming version of rustc *regardless of which edition the code uses*. This is the meaning that rustc exclusively exposes to users as "future incompatible". -* the user has written code that will either no longer compiler *or* will change +* The user has written code that will either no longer compiler *or* will change meaning in an upcoming *edition*. These are often called "edition lints" and can be typically seen in the various "edition compatibility" lint groups (e.g., `rust_2021_compatibility`) that are used to lint against code that will break if the user updates the crate's edition. @@ -663,7 +741,7 @@ declare_lint! { Notice the `reason` field which describes why the future incompatible change is happening. This will change the diagnostic message the user receives as well as determine which lint groups the lint is added to. In the example above, the lint is an "edition lint" -(since it's "reason" is `EditionError`) signifying to the user that the use of anonymous +(since its "reason" is `EditionError`), signifying to the user that the use of anonymous parameters will no longer compile in Rust 2018 and beyond. Inside [LintStore::register_lints][fi-lint-groupings], lints with `future_incompatible` diff --git a/src/diagnostics/diagnostic-codes.md b/src/diagnostics/diagnostic-codes.md deleted file mode 100644 index 3618b43cd..000000000 --- a/src/diagnostics/diagnostic-codes.md +++ /dev/null @@ -1,79 +0,0 @@ -# Diagnostic codes -We generally try to assign each error message a unique code like `E0123`. These -codes are defined in the compiler in the `diagnostics.rs` files found in each -crate, which basically consist of macros. The codes come in two varieties: those -that have an extended write-up, and those that do not. Whenever possible, if you -are making a new code, you should write an extended write-up. - -### Allocating a fresh code - -Error codes are stored in `compiler/rustc_error_codes`. - -To create a new error, you first need to find the next available -code. You can find it with `tidy`: - -``` -./x.py test tidy -``` - -This will invoke the tidy script, which generally checks that your code obeys -our coding conventions. One of those jobs is to check that diagnostic codes are -indeed unique. Once it is finished with that, tidy will print out the lowest -unused code: - -``` -... -tidy check (x86_64-apple-darwin) -* 470 error codes -* highest error code: E0591 -... -``` - -Here we see the highest error code in use is `E0591`, so we _probably_ want -`E0592`. To be sure, run `rg E0592` and check, you should see no references. - -Ideally, you will write an extended description for your error, -which will go in `rustc_error_codes/src/error_codes/E0592.md`. -To register the error, open `rustc_error_codes/src/error_codes.rs` and add the -code (in its proper numerical order) into` register_diagnostics!` macro, like -this: - -```rust -register_diagnostics! { - ... - E0592: include_str!("./error_codes/E0592.md"), -} -``` - -But you can also add it without an extended description: - -```rust -register_diagnostics! { - ... - E0592, // put a description here -} -``` - -To actually issue the error, you can use the `struct_span_err!` macro: - -```rust -struct_span_err!(self.tcx.sess, // some path to the session here - span, // whatever span in the source you want - E0592, // your new error code - fluent::example::an_error_message) - .emit() // actually issue the error -``` - -If you want to add notes or other snippets, you can invoke methods before you -call `.emit()`: - -```rust -struct_span_err!(...) - .span_label(another_span, fluent::example::example_label) - .span_note(another_span, fluent::example::separate_note) - .emit_() -``` - -For an example of a PR adding an error code, see [#76143]. - -[#76143]: https://github.com/rust-lang/rust/pull/76143 diff --git a/src/diagnostics/diagnostic-items.md b/src/diagnostics/diagnostic-items.md index dcaba533e..78fdd032d 100644 --- a/src/diagnostics/diagnostic-items.md +++ b/src/diagnostics/diagnostic-items.md @@ -48,7 +48,7 @@ A new diagnostic item can be added with these two steps: For the naming conventions of diagnostic items, please refer to [*Naming Conventions*](#naming-conventions). -2. +2. Diagnostic items in code are accessed via symbols in [`rustc_span::symbol::sym`]. To add your newly-created diagnostic item, diff --git a/src/diagnostics/diagnostic-structs.md b/src/diagnostics/diagnostic-structs.md index e26ba5f34..6d698e3f3 100644 --- a/src/diagnostics/diagnostic-structs.md +++ b/src/diagnostics/diagnostic-structs.md @@ -23,7 +23,7 @@ pub struct FieldAlreadyDeclared { #[primary_span] #[label] pub span: Span, - #[label(hir_analysis_previous_decl_label)] + #[label(previous_decl_label)] pub prev_span: Span, } ``` @@ -70,7 +70,7 @@ Diagnostics are more than just their primary message, they often include labels, notes, help messages and suggestions, all of which can also be specified on a `Diagnostic`. -`#[label]`, `#[help]` and `#[note]` can all be applied to fields which have the +`#[label]`, `#[help]`, `#[warning]` and `#[note]` can all be applied to fields which have the type `Span`. Applying any of these attributes will create the corresponding subdiagnostic with that `Span`. These attributes will look for their diagnostic message in a Fluent attribute attached to the primary Fluent @@ -82,16 +82,16 @@ these attributes can also take a value that is the attribute name to look for Other types have special behavior when used in a `Diagnostic` derive: -- Any attribute applied to an `Option` and will only emit a +- Any attribute applied to an `Option` will only emit a subdiagnostic if the option is `Some(..)`. - Any attribute applied to a `Vec` will be repeated for each element of the vector. -`#[help]` and `#[note]` can also be applied to the struct itself, in which case +`#[help]`, `#[warning]` and `#[note]` can also be applied to the struct itself, in which case they work exactly like when applied to fields except the subdiagnostic won't have a `Span`. These attributes can also be applied to fields of type `()` for the same effect, which when combined with the `Option` type can be used to -represent optional `#[note]`/`#[help]` subdiagnostics. +represent optional `#[note]`/`#[help]`/`#[warning]` subdiagnostics. Suggestions can be emitted using one of four field attributes: @@ -161,14 +161,14 @@ following attributes: - `code = "..."` (_Optional_) - Specifies the error code. - `#[note]` or `#[note(slug)]` (_Optional_) - - _Applied to struct or `Span`/`()` fields._ + - _Applied to struct or struct fields of type `Span`, `Option<()>` or `()`._ - Adds a note subdiagnostic. - Value is a path to an item in `rustc_errors::fluent` for the note's message. - Defaults to equivalent of `.note`. - If applied to a `Span` field, creates a spanned note. - `#[help]` or `#[help(slug)]` (_Optional_) - - _Applied to struct or `Span`/`()` fields._ + - _Applied to struct or struct fields of type `Span`, `Option<()>` or `()`._ - Adds a help subdiagnostic. - Value is a path to an item in `rustc_errors::fluent` for the note's message. @@ -180,8 +180,8 @@ following attributes: - Value is a path to an item in `rustc_errors::fluent` for the note's message. - Defaults to equivalent of `.label`. -- `#[warn_]` or `#[warn_(slug)]` (_Optional_) - - _Applied to `Span` fields._ +- `#[warning]` or `#[warning(slug)]` (_Optional_) + - _Applied to struct or struct fields of type `Span`, `Option<()>` or `()`._ - Adds a warning subdiagnostic. - Value is a path to an item in `rustc_errors::fluent` for the note's message. @@ -253,6 +253,7 @@ attribute applied to the struct or each variant, one of: - `#[label(..)]` for defining a label - `#[note(..)]` for defining a note - `#[help(..)]` for defining a help +- `#[warning(..)]` for defining a warning - `#[suggestion{,_hidden,_short,_verbose}(..)]` for defining a suggestion All of the above must provide a slug as the first positional argument (a path @@ -333,7 +334,7 @@ diagnostic struct. ### Reference `#[derive(Subdiagnostic)]` supports the following attributes: -- `#[label(slug)]`, `#[help(slug)]` or `#[note(slug)]` +- `#[label(slug)]`, `#[help(slug)]`, `#[warning(slug)]` or `#[note(slug)]` - _Applied to struct or enum variant. Mutually exclusive with struct/enum variant attributes._ - _Mandatory_ - Defines the type to be representing a label, help or note. diff --git a/src/diagnostics/error-codes.md b/src/diagnostics/error-codes.md new file mode 100644 index 000000000..7cdf5ebca --- /dev/null +++ b/src/diagnostics/error-codes.md @@ -0,0 +1,95 @@ +# Error codes +We generally try to assign each error message a unique code like `E0123`. These +codes are defined in the compiler in the `diagnostics.rs` files found in each +crate, which basically consist of macros. All error codes have an associated +explanation: new error codes must include them. Note that not all _historical_ +(no longer emitted) error codes have explanations. + +## Error explanations + +The explanations are written in Markdown (see the [CommonMark Spec] for +specifics around syntax), and all of them are linked in the [`rustc_error_codes`] +crate. Please read [RFC 1567] for details on how to format and write long error +codes. As of February 2023, there is an +effort[^new-explanations] to replace this largely outdated RFC with a new more +flexible standard. + +Error explanations should expand on the error message and provide details about +_why_ the error occurs. It is not helpful for users to copy-paste a quick fix; +explanations should help users understand why their code cannot be accepted by +the compiler. Rust prides itself on helpful error messages and long-form +explanations are no exception. However, before error explanations are +overhauled[^new-explanations] it is a bit open as to how exactly they should be +written, as always: ask your reviewer or ask around on the Rust Discord or Zulip. + +[^new-explanations]: See the draft RFC [here][new-explanations-rfc]. + +[`rustc_error_codes`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_error_codes/error_codes/index.html +[CommonMark Spec]: https://spec.commonmark.org/current/ +[RFC 1567]: https://github.com/rust-lang/rfcs/blob/master/text/1567-long-error-codes-explanation-normalization.md +[new-explanations-rfc]: https://github.com/rust-lang/rfcs/pull/3370 + +## Allocating a fresh code + +Error codes are stored in `compiler/rustc_error_codes`. + +To create a new error, you first need to find the next available +code. You can find it with `tidy`: + +``` +./x test tidy +``` + +This will invoke the tidy script, which generally checks that your code obeys +our coding conventions. Some of these jobs check error codes and ensure that +there aren't duplicates, etc (the tidy check is defined in +`src/tools/tidy/src/error_codes.rs`). Once it is finished with that, tidy will +print out the highest used error code: + +``` +... +tidy check +Found 505 error codes +Highest error code: `E0591` +... +``` + +Here we see the highest error code in use is `E0591`, so we _probably_ want +`E0592`. To be sure, run `rg E0592` and check, you should see no references. + +You will have to write an extended description for your error, +which will go in `rustc_error_codes/src/error_codes/E0592.md`. +To register the error, open `rustc_error_codes/src/error_codes.rs` and add the +code (in its proper numerical order) into` register_diagnostics!` macro, like +this: + +```rust +register_diagnostics! { + ... + E0592: include_str!("./error_codes/E0592.md"), +} +``` + +To actually issue the error, you can use the `struct_span_err!` macro: + +```rust +struct_span_err!(self.tcx.sess, // some path to the session here + span, // whatever span in the source you want + E0592, // your new error code + fluent::example::an_error_message) + .emit() // actually issue the error +``` + +If you want to add notes or other snippets, you can invoke methods before you +call `.emit()`: + +```rust +struct_span_err!(...) + .span_label(another_span, fluent::example::example_label) + .span_note(another_span, fluent::example::separate_note) + .emit() +``` + +For an example of a PR adding an error code, see [#76143]. + +[#76143]: https://github.com/rust-lang/rust/pull/76143 diff --git a/src/diagnostics/translation.md b/src/diagnostics/translation.md index e36333039..a42a15484 100644 --- a/src/diagnostics/translation.md +++ b/src/diagnostics/translation.md @@ -86,10 +86,10 @@ excellent examples of translating messages into different locales and the information that needs to be provided by the code to do so. ### Compile-time validation and typed identifiers -rustc's Fluent resources for the default locale (`en-US`) are in the -[`compiler/rustc_error_messages/locales/en-US`] directory. Currently, each crate -which defines translatable diagnostics has its own Fluent resource, such as -`parser.ftl` or `typeck.ftl`. +Currently, each crate which defines translatable diagnostics has its own +Fluent resource in a file named `messages.ftl`, such as +[`compiler/rustc_borrowck/messages.ftl`] and +[`compiler/rustc_parse/messages.ftl`]. rustc's `fluent_messages` macro performs compile-time validation of Fluent resources and generates code to make it easier to refer to Fluent messages in @@ -222,7 +222,7 @@ returned by `Emitter::fluent_bundle`. This bundle is used preferentially when translating messages, the fallback bundle is only used if the primary bundle is missing a message or not provided. -As of June 2022, there are no locale bundles +As of Jan 2023, there are no locale bundles distributed with the compiler, but mechanisms are implemented for loading bundles. @@ -240,5 +240,6 @@ won't fail. Bundle loading can fail if a requested locale is missing, Fluent files are malformed, or a message is duplicated in multiple resources. [Fluent]: https://projectfluent.org -[`compiler/rustc_error_messages/locales/en-US`]: https://github.com/rust-lang/rust/tree/master/compiler/rustc_error_messages/locales/en-US +[`compiler/rustc_borrowck/messages.ftl`]: https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_borrowck/messages.ftl +[`compiler/rustc_parse/messages.ftl`]: https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_parse/messages.ftl [`rustc_error_messages::DiagnosticMessage`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_error_messages/enum.DiagnosticMessage.html diff --git a/src/early-late-bound-summary.md b/src/early-late-bound-summary.md new file mode 100644 index 000000000..223e2251d --- /dev/null +++ b/src/early-late-bound-summary.md @@ -0,0 +1,10 @@ +# Early/Late bound parameters + +This section discusses what it means for generic parameters to be early or late bound. + +```rust +fn foo<'a, T>(b: &'a u32) -> &'a u32 { a } +// ^^ ^early bound +// ^^ +// ^^late bound +``` \ No newline at end of file diff --git a/src/effects.md b/src/effects.md new file mode 100644 index 000000000..1fda7bcbb --- /dev/null +++ b/src/effects.md @@ -0,0 +1,66 @@ +# Effects and effect checking + +Note: all of this describes the implementation of the unstable `effects` and +`const_trait_impl` features. None of this implementation is usable or visible from +stable Rust. + +The implementation of const traits and `~const` bounds is a limited effect system. +It is used to allow trait bounds on `const fn` to be used within the `const fn` for +method calls. Within the function, in order to know whether a method on a trait +bound is `const`, we need to know whether there is a `~const` bound for the trait. +In order to know whether we can instantiate a `~const` bound on a `const fn`, we +need to know whether there is a `const_trait` impl for the type and trait being +used (or whether the `const fn` is used at runtime, then any type implementing the +trait is ok, just like with other bounds). + +We perform these checks via a const generic boolean that gets attached to all +`const fn` and `const trait`. The following sections will explain the desugarings +and the way we perform the checks at call sites. + +The const generic boolean is inverted to the meaning of `const`. In the compiler +it is called `host`, because it enables "host APIs" like `static` items, network +access, disk access, random numbers and everything else that isn't available in +`const` contexts. So `false` means "const", `true` means "not const" and if it's +a generic parameter, it means "maybe const" (meaning we're in a const fn or const +trait). + +## `const fn` + +All `const fn` have a `#[rustc_host] const host: bool` generic parameter that is +hidden from users. Any `~const Trait` bounds in the generics list or `where` bounds +of a `const fn` get converted to `Trait + Trait` bounds. The `Trait` +exists so that associated types of the generic param can be used from projections +like `::Assoc`, because there are no `` projections for now. + +## `#[const_trait] trait`s + +The `#[const_trait]` attribute gives the marked trait a `#[rustc_host] const host: bool` +generic parameter. All functions of the trait "inherit" this generic parameter, just like +they have all the regular generic parameters of the trait. Any `~const Trait` super-trait +bounds get desugared to `Trait + Trait` in order to allow using associated +types and consts of the super traits in the trait declaration. This is necessary, because +`::Assoc` is always `>::Assoc` as there is +no `` syntax. + +## `typeck` performing method and function call checks. + +When generic parameters are instantiated for any items, the `host` generic parameter +is always instantiated as an inference variable. This is a special kind of inference var +that is not part of the type or const inference variables, similar to how we have +special inference variables for type variables that we know to be an integer, but not +yet which one. These separate inference variables fall back to `true` at +the end of typeck (in `fallback_effects`) to ensure that `let _ = some_fn_item_name;` +will keep compiling. + +All actually used (in function calls, casts, or anywhere else) function items, will +have the `enforce_context_effects` method invoked. +It trivially returns if the function being called has no `host` generic parameter. + +In order to error if a non-const function is called in a const context, we have not +yet disabled the const-check logic that happens on MIR, because +`enforce_context_effects` does not yet perform this check. + +The function call's `host` parameter is then equated to the context's `host` value, +which almost always trivially succeeds, as it was an inference var. If the inference +var has already been bound (since the function item is invoked twice), the second +invocation checks it against the first. diff --git a/src/external-repos.md b/src/external-repos.md new file mode 100644 index 000000000..533f7eb5e --- /dev/null +++ b/src/external-repos.md @@ -0,0 +1,113 @@ +# Using External Repositories + +The `rust-lang/rust` git repository depends on several other repos in the `rust-lang` organization. +There are three main ways we use dependencies: +1. As a Cargo dependency through crates.io (e.g. `rustc-rayon`) +2. As a git subtree (e.g. `clippy`) +3. As a git submodule (e.g. `cargo`) + +As a general rule, use crates.io for libraries that could be useful for others in the ecosystem; use +subtrees for tools that depend on compiler internals and need to be updated if there are breaking +changes; and use submodules for tools that are independent of the compiler. + +## External Dependencies (subtree) + +As a developer to this repository, you don't have to treat the following external projects +differently from other crates that are directly in this repo: + +* [Clippy](https://github.com/rust-lang/rust-clippy) +* [Miri] +* [rustfmt](https://github.com/rust-lang/rustfmt) +* [rust-analyzer](https://github.com/rust-lang/rust-analyzer) + +[Miri]: https://github.com/rust-lang/miri + +In contrast to `submodule` dependencies +(see below for those), the `subtree` dependencies are just regular files and directories which can +be updated in tree. However, if possible, enhancements, bug fixes, etc. specific +to these tools should be filed against the tools directly in their respective +upstream repositories. The exception is that when rustc changes are required to +implement a new tool feature or test, that should happen in one collective rustc PR. + +### Synchronizing a subtree + +Periodically the changes made to subtree based dependencies need to be synchronized between this +repository and the upstream tool repositories. + +Subtree synchronizations are typically handled by the respective tool maintainers. Other users +are welcome to submit synchronization PRs, however, in order to do so you will need to modify +your local git installation and follow a very precise set of instructions. +These instructions are documented, along with several useful tips and tricks, in the +[syncing subtree changes][clippy-sync-docs] section in Clippy's Contributing guide. +The instructions are applicable for use with any subtree based tool, just be sure to +use the correct corresponding subtree directory and remote repository. + +The synchronization process goes in two directions: `subtree push` and `subtree pull`. + +A `subtree push` takes all the changes that happened to the copy in this repo and creates commits +on the remote repo that match the local changes. Every local +commit that touched the subtree causes a commit on the remote repo, but +is modified to move the files from the specified directory to the tool repo root. + +A `subtree pull` takes all changes since the last `subtree pull` +from the tool repo and adds these commits to the rustc repo along with a merge commit that moves +the tool changes into the specified directory in the Rust repository. + +It is recommended that you always do a push first and get that merged to the tool master branch. +Then, when you do a pull, the merge works without conflicts. +While it's definitely possible to resolve conflicts during a pull, you may have to redo the conflict +resolution if your PR doesn't get merged fast enough and there are new conflicts. Do not try to +rebase the result of a `git subtree pull`, rebasing merge commits is a bad idea in general. + +You always need to specify the `-P` prefix to the subtree directory and the corresponding remote +repository. If you specify the wrong directory or repository +you'll get very fun merges that try to push the wrong directory to the wrong remote repository. +Luckily you can just abort this without any consequences by throwing away either the pulled commits +in rustc or the pushed branch on the remote and try again. It is usually fairly obvious +that this is happening because you suddenly get thousands of commits that want to be synchronized. + +[clippy-sync-docs]: https://doc.rust-lang.org/nightly/clippy/development/infrastructure/sync.html + +### Creating a new subtree dependency + +If you want to create a new subtree dependency from an existing repository, call (from this +repository's root directory!) + +``` +git subtree add -P src/tools/clippy https://github.com/rust-lang/rust-clippy.git master +``` + +This will create a new commit, which you may not rebase under any circumstances! Delete the commit +and redo the operation if you need to rebase. + +Now you're done, the `src/tools/clippy` directory behaves as if Clippy were +part of the rustc monorepo, so no one but you (or others that synchronize +subtrees) actually needs to use `git subtree`. + + +## External Dependencies (submodules) + +Building Rust will also use external git repositories tracked using [git +submodules]. The complete list may be found in the [`.gitmodules`] file. Some +of these projects are required (like `stdarch` for the standard library) and +some of them are optional (like `src/doc/book`). + +Usage of submodules is discussed more in the [Using Git chapter](git.md#git-submodules). + +Some of the submodules are allowed to be in a "broken" state where they +either don't build or their tests don't pass, e.g. the documentation books +like [The Rust Reference]. Maintainers of these projects will be notified +when the project is in a broken state, and they should fix them as soon +as possible. The current status is tracked on the [toolstate website]. +More information may be found on the Forge [Toolstate chapter]. +In practice, it is very rare for documentation to have broken toolstate. + +Breakage is not allowed in the beta and stable channels, and must be addressed +before the PR is merged. They are also not allowed to be broken on master in +the week leading up to the beta cut. + +[git submodules]: https://git-scm.com/book/en/v2/Git-Tools-Submodules +[`.gitmodules`]: https://github.com/rust-lang/rust/blob/master/.gitmodules +[The Rust Reference]: https://github.com/rust-lang/reference/ +[toolstate website]: https://rust-lang-nursery.github.io/rust-toolstate/ +[Toolstate chapter]: https://forge.rust-lang.org/infra/toolstate.html diff --git a/src/feature-gates.md b/src/feature-gates.md index 9e9a83ea6..788f93d66 100644 --- a/src/feature-gates.md +++ b/src/feature-gates.md @@ -3,44 +3,16 @@ This chapter is intended to provide basic help for adding, removing, and modifying feature gates. +Note that this is specific to *language* feature gates; *library* feature gates use [a different +mechanism][libs-gate]. -## Adding a feature gate - -See ["Stability in code"] for help with adding a new feature; this section just -covers how to add the feature gate *declaration*. - -First, add the feature name to `rustc_span/src/symbol.rs` in the `Symbols {...}` block. - -Then, add a feature gate declaration to `rustc_feature/src/active.rs` in the active -`declare_features` block: - -```rust,ignore -/// description of feature -(active, $feature_name, "$current_nightly_version", Some($tracking_issue_number), $edition) -``` +[libs-gate]: ./stability.md -where `$edition` has the type `Option`, and is typically -just `None`. - -For example: - -```rust,ignore -/// Allows defining identifiers beyond ASCII. -(active, non_ascii_idents, "1.0.0", Some(55467), None), -``` - -Features can be marked as incomplete, and trigger the warn-by-default [`incomplete_features` lint] -by setting their type to `incomplete`: - -```rust,ignore -/// Allows unsized rvalues at arguments and parameters. -(incomplete, unsized_locals, "1.30.0", Some(48055), None), -``` +## Adding a feature gate -When added, the current version should be the one for the current nightly. -Once the feature is moved to `accepted.rs`, the version is changed to that -nightly version. +See ["Stability in code"][adding] in the "Implementing new features" section for instructions. +[adding]: ./implementing_new_features.md#stability-in-code ## Removing a feature gate @@ -48,12 +20,12 @@ nightly version. To remove a feature gate, follow these steps: -1. Remove the feature gate declaration in `rustc_feature/src/active.rs`. +1. Remove the feature gate declaration in `rustc_feature/src/unstable.rs`. It will look like this: ```rust,ignore /// description of feature - (active, $feature_name, "$version", Some($tracking_issue_number), $edition) + (unstable, $feature_name, "$version", Some($tracking_issue_number), $edition) ``` 2. Add a modified version of the feature gate declaration that you just @@ -73,12 +45,12 @@ To remove a feature gate, follow these steps: To rename a feature gate, follow these steps (the first two are the same steps to follow when [removing a feature gate][removing]): -1. Remove the old feature gate declaration in `rustc_feature/src/active.rs`. +1. Remove the old feature gate declaration in `rustc_feature/src/unstable.rs`. It will look like this: ```rust,ignore /// description of feature - (active, $old_feature_name, "$version", Some($tracking_issue_number), $edition) + (unstable, $old_feature_name, "$version", Some($tracking_issue_number), $edition) ``` 2. Add a modified version of the old feature gate declaration that you just @@ -92,12 +64,12 @@ to follow when [removing a feature gate][removing]): ``` 3. Add a feature gate declaration with the new name to - `rustc_feature/src/active.rs`. It should look very similar to the old + `rustc_feature/src/unstable.rs`. It should look very similar to the old declaration: ```rust,ignore /// description of feature - (active, $new_feature_name, "$version", Some($tracking_issue_number), $edition) + (unstable, $new_feature_name, "$version", Some($tracking_issue_number), $edition) ``` @@ -109,5 +81,4 @@ updating the declaration! ["Stability in code"]: ./implementing_new_features.md#stability-in-code -[`incomplete_features` lint]: https://doc.rust-lang.org/rustc/lints/listing/warn-by-default.html#incomplete-features ["Updating the feature-gate listing"]: ./stabilization_guide.md#updating-the-feature-gate-listing diff --git a/src/fuzzing.md b/src/fuzzing.md new file mode 100644 index 000000000..869fc2f71 --- /dev/null +++ b/src/fuzzing.md @@ -0,0 +1,149 @@ +# Fuzzing + + + +For the purposes of this guide, *fuzzing* is any testing methodology that +involves compiling a wide variety of programs in an attempt to uncover bugs in +rustc. Fuzzing is often used to find internal compiler errors (ICEs). Fuzzing +can be beneficial, because it can find bugs before users run into them and +provide small, self-contained programs that make the bug easier to track down. +However, some common mistakes can reduce the helpfulness of fuzzing and end up +making contributors' lives harder. To maximize your positive impact on the Rust +project, please read this guide before reporting fuzzer-generated bugs! + +## Guidelines + +### In a nutshell + +*Please do:* + +- Ensure the bug is still present on the latest nightly rustc +- Include a reasonably minimal, standalone example along with any bug report +- Include all of the information requested in the bug report template +- Search for existing reports with the same message and query stack +- Format the test case with `rustfmt`, if it maintains the bug +- Indicate that the bug was found by fuzzing + +*Please don't:* + +- Don't report lots of bugs that use internal features, including but not + limited to `custom_mir`, `lang_items`, `no_core`, and `rustc_attrs`. +- Don't seed your fuzzer with inputs that are known to crash rustc (details + below). + +### Discussion + +If you're not sure whether or not an ICE is a duplicate of one that's already +been reported, please go ahead and report it and link to issues you think might +be related. In general, ICEs on the same line but with different *query stacks* +are usually distinct bugs. For example, [#109020][#109020] and [#109129][#109129] +had similar error messages: + +``` +error: internal compiler error: compiler/rustc_middle/src/ty/normalize_erasing_regions.rs:195:90: Failed to normalize <[closure@src/main.rs:36:25: 36:28] as std::ops::FnOnce<(Emplacable<()>,)>>::Output, maybe try to call `try_normalize_erasing_regions` instead +``` +``` +error: internal compiler error: compiler/rustc_middle/src/ty/normalize_erasing_regions.rs:195:90: Failed to normalize <() as Project>::Assoc, maybe try to call `try_normalize_erasing_regions` instead +``` +but different query stacks: +``` +query stack during panic: +#0 [fn_abi_of_instance] computing call ABI of `<[closure@src/main.rs:36:25: 36:28] as core::ops::function::FnOnce<(Emplacable<()>,)>>::call_once - shim(vtable)` +end of query stack +``` +``` +query stack during panic: +#0 [check_mod_attrs] checking attributes in top-level module +#1 [analysis] running analysis passes on this crate +end of query stack +``` + +[#109020]: https://github.com/rust-lang/rust/issues/109020 +[#109129]: https://github.com/rust-lang/rust/issues/109129 + +## Building a corpus + +When building a corpus, be sure to avoid collecting tests that are already +known to crash rustc. A fuzzer that is seeded with such tests is more likely to +generate bugs with the same root cause, wasting everyone's time. The simplest +way to avoid this is to loop over each file in the corpus, see if it causes an +ICE, and remove it if so. + +To build a corpus, you may want to use: + +- The rustc/rust-analyzer/clippy test suites (or even source code) --- though avoid + tests that are already known to cause failures, which often begin with comments + like `// failure-status: 101` or `// known-bug: #NNN`. +- The already-fixed ICEs in [Glacier][glacier] --- though avoid the unfixed + ones in `ices/`! + +## Extra credit + +Here are a few things you can do to help the Rust project after filing an ICE. + +- [Bisect][bisect] the bug to figure out when it was introduced +- Fix "distractions": problems with the test case that don't contribute to + triggering the ICE, such as syntax errors or borrow-checking errors +- Minimize the test case (see below) +- Add the minimal test case to [Glacier][glacier] + +[bisect]: https://rust-lang.github.io/cargo-bisect-rustc/ + +## Minimization + +It is helpful to carefully *minimize* the fuzzer-generated input. When +minimizing, be careful to preserve the original error, and avoid introducing +distracting problems such as syntax, type-checking, or borrow-checking errors. + +There are some tools that can help with minimization. If you're not sure how +to avoid introducing syntax, type-, and borrow-checking errors while using +these tools, post both the complete and minimized test cases. Generally, +*syntax-aware* tools give the best results in the least amount of time. +[`treereduce-rust`][treereduce] and [picireny][picireny] are syntax-aware. +[`halfempty`][halfempty] is not, but is generally a high-quality tool. + +[halfempty]: https://github.com/googleprojectzero/halfempty +[picireny]: https://github.com/renatahodovan/picireny +[treereduce]: https://github.com/langston-barrett/treereduce + +## Effective fuzzing + +When fuzzing rustc, you may want to avoid generating machine code, since this +is mostly done by LLVM. Try `--emit=mir` instead. + +A variety of compiler flags can uncover different issues. `-Zmir-opt-level=4` +will turn on MIR optimization passes that are not run by default, potentially +uncovering interesting bugs. `-Zvalidate-mir` can help uncover such bugs. + +If you're fuzzing a compiler you built, you may want to build it with `-C +target-cpu=native` or even PGO/BOLT to squeeze out a few more executions per +second. Of course, it's best to try multiple build configurations and see +what actually results in superior throughput. + +You may want to build rustc from source with debug assertions to find +additional bugs, though this is a trade-off: it can slow down fuzzing by +requiring extra work for every execution. To enable debug assertions, add this +to `config.toml` when compiling rustc: + +```toml +[rust] +debug-assertions = true +``` + +ICEs that require debug assertions to reproduce should be tagged +[`requires-debug-assertions`][requires-debug-assertions]. + +[requires-debug-assertions]: https://github.com/rust-lang/rust/labels/requires-debug-assertions + +## Existing projects + +- [fuzz-rustc][fuzz-rustc] demonstrates how to fuzz rustc with libfuzzer +- [icemaker][icemaker] runs rustc and other tools on a large number of source + files with a variety of flags to catch ICEs +- [tree-splicer][tree-splicer] generates new source files by combining existing + ones while maintaining correct syntax + +[glacier]: https://github.com/rust-lang/glacier +[fuzz-rustc]: https://github.com/dwrensha/fuzz-rustc +[icemaker]: https://github.com/matthiaskrgr/icemaker/ +[tree-splicer]: https://github.com/langston-barrett/tree-splicer/ diff --git a/src/generic_arguments.md b/src/generic_arguments.md index c9911acf5..6e09e8620 100644 --- a/src/generic_arguments.md +++ b/src/generic_arguments.md @@ -1,28 +1,28 @@ # Generic arguments -A `ty::subst::GenericArg<'tcx>` represents some entity in the type system: a type +A `ty::GenericArg<'tcx>` represents some entity in the type system: a type (`Ty<'tcx>`), lifetime (`ty::Region<'tcx>`) or constant (`ty::Const<'tcx>`). -`GenericArg` is used to perform substitutions of generic parameters for concrete +`GenericArg` is used to perform instantiation of generic parameters to concrete arguments, such as when calling a function with generic parameters explicitly -with type arguments. Substitutions are represented using the -[`Subst` type](#subst) as described below. +with type arguments. Instantiations are represented using the +[`GenericArgs` type](#genericargs) as described below. -## `Subst` -`ty::subst::Subst<'tcx>` is intuitively simply a slice of `GenericArg<'tcx>`s, -acting as an ordered list of substitutions from generic parameters to +## `GenericArgs` +`ty::GenericArgs<'tcx>` is intuitively simply a slice of `GenericArg<'tcx>`s, +acting as an ordered list of generic parameters instantiated to concrete arguments (such as types, lifetimes and consts). For example, given a `HashMap` with two type parameters, `K` and `V`, an instantiation of the parameters, for example `HashMap`, would be -represented by the substitution `&'tcx [tcx.types.i32, tcx.types.u32]`. +represented by `&'tcx [tcx.types.i32, tcx.types.u32]`. -`Subst` provides various convenience methods to instantiate substitutions +`GenericArgs` provides various convenience methods to instantiate generic arguments given item definitions, which should generally be used rather than explicitly -constructing such substitution slices. +instantiating such slices. ## `GenericArg` The actual `GenericArg` struct is optimised for space, storing the type, lifetime or const as an interned pointer containing a tag identifying its kind (in the -lowest 2 bits). Unless you are working with the `Subst` implementation +lowest 2 bits). Unless you are working with the `GenericArgs` implementation specifically, you should generally not have to deal with `GenericArg` and instead make use of the safe [`GenericArgKind`](#genericargkind) abstraction. @@ -30,7 +30,7 @@ make use of the safe [`GenericArgKind`](#genericargkind) abstraction. As `GenericArg` itself is not type-safe, the `GenericArgKind` enum provides a more convenient and safe interface for dealing with generic arguments. An `GenericArgKind` can be converted to a raw `GenericArg` using `GenericArg::from()` -(or simply `.into()` when the context is clear). As mentioned earlier, substitution +(or simply `.into()` when the context is clear). As mentioned earlier, instantiation lists store raw `GenericArg`s, so before dealing with them, it is preferable to convert them to `GenericArgKind`s first. This is done by calling the `.unpack()` method. @@ -44,7 +44,7 @@ fn deal_with_generic_arg<'tcx>(generic_arg: GenericArg<'tcx>) -> GenericArg<'tcx GenericArgKind::Lifetime(lt) => { /* ... */ } GenericArgKind::Const(ct) => { /* ... */ } }; - // Pack the `GenericArgKind` to store it in a substitution list. + // Pack the `GenericArgKind` to store it in a generic args list. new_generic_arg.into() } ``` diff --git a/src/generics.md b/src/generics.md index 0173bee8f..6c61ab87f 100644 --- a/src/generics.md +++ b/src/generics.md @@ -1,4 +1,4 @@ -# Generics and substitutions +# Generics and GenericArgs Given a generic type `MyType`, we may want to swap out the generics `A, B, …` for some other types (possibly other generics or concrete types). We do this a lot while doing type @@ -6,18 +6,20 @@ inference, type checking, and trait solving. Conceptually, during these routines that one type is equal to another type and want to swap one out for the other and then swap that out for another type and so on until we eventually get some concrete types (or an error). -In rustc this is done using the `SubstsRef` that we mentioned above (“substs” = “substitutions”). -Conceptually, you can think of `SubstsRef` as a list of types that are to be substituted for the -generic type parameters of the ADT. +In rustc this is done using [GenericArgsRef]. +Conceptually, you can think of `GenericArgsRef` as a list of types that are to be substituted for + the generic type parameters of the ADT. -`SubstsRef` is a type alias of `&'tcx List>` (see [`List` rustdocs][list]). +`GenericArgsRef` is a type alias of `&'tcx List>` (see [`List` rustdocs][list]). [`GenericArg`] is essentially a space-efficient wrapper around [`GenericArgKind`], which is an enum -indicating what kind of generic the type parameter is (type, lifetime, or const). Thus, `SubstsRef` -is conceptually like a `&'tcx [GenericArgKind<'tcx>]` slice (but it is actually a `List`). +indicating what kind of generic the type parameter is (type, lifetime, or const). +Thus, `GenericArgsRef` is conceptually like a `&'tcx [GenericArgKind<'tcx>]` slice (but it is +actually a `List`). [list]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.List.html -[`GenericArg`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/subst/struct.GenericArg.html -[`GenericArgKind`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/subst/enum.GenericArgKind.html +[`GenericArg`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.GenericArg.html +[`GenericArgKind`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.GenericArgKind.html +[GenericArgsRef]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/type.GenericArgsRef.html So why do we use this `List` type instead of making it really a slice? It has the length "inline", so `&List` is only 32 bits. As a consequence, it cannot be "subsliced" (that only works if the @@ -35,10 +37,10 @@ struct MyStruct - There would be an `AdtDef` (and corresponding `DefId`) for `MyStruct`. - There would be a `TyKind::Param` (and corresponding `DefId`) for `T` (more later). -- There would be a `SubstsRef` containing the list `[GenericArgKind::Type(Ty(T))]` +- There would be a `GenericArgsRef` containing the list `[GenericArgKind::Type(Ty(T))]` - The `Ty(T)` here is my shorthand for entire other `ty::Ty` that has `TyKind::Param`, which we mentioned in the previous point. -- This is one `TyKind::Adt` containing the `AdtDef` of `MyStruct` with the `SubstsRef` above. +- This is one `TyKind::Adt` containing the `AdtDef` of `MyStruct` with the `GenericArgsRef` above. Finally, we will quickly mention the [`Generics`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Generics.html) type. It @@ -112,33 +114,31 @@ This example has a few different substitutions: Let’s look a bit more closely at that last substitution to see why we use indexes. If we want to find the type of `foo.x`, we can get generic type of `x`, which is `Vec`. Now we can take -the index `0` and use it to find the right type substitution: looking at `Foo`'s `SubstsRef`, we -have the list `[u32, f32]` , since we want to replace index `0`, we take the 0-th index of this +the index `0` and use it to find the right type substitution: looking at `Foo`'s `GenericArgsRef`, +we have the list `[u32, f32]` , since we want to replace index `0`, we take the 0-th index of this list, which is `u32`. Voila! You may have a couple of followup questions… - **`type_of`** How do we get the “generic type of `x`"? You can get the type of pretty much anything + **`type_of`** How do we get the "generic type of `x`"? You can get the type of pretty much anything with the `tcx.type_of(def_id)` query. In this case, we would pass the `DefId` of the field `x`. The `type_of` query always returns the definition with the generics that are in scope of the definition. For example, `tcx.type_of(def_id_of_my_struct)` would return the “self-view” of `MyStruct`: `Adt(Foo, &[Param(0), Param(1)])`. -**`subst`** How do we actually do the substitutions? There is a function for that too! You use -[`subst`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/subst/struct.EarlyBinder.html#method.subst) to -replace a `SubstRef` with another list of types. +How do we actually do the substitutions? There is a function for that too! You +use [`instantiate`] to replace a `GenericArgsRef` with another list of types. -[Here is an example of actually using `subst` in the compiler][substex]. The exact details are not -too important, but in this piece of code, we happen to be converting from the `rustc_hir::Ty` to -a real `ty::Ty`. You can see that we first get some substitutions (`substs`). Then we call -`type_of` to get a type and call `ty.subst(substs)` to get a new version of `ty` with -the substitutions made. +[Here is an example of actually using `instantiate` in the compiler][instantiatex]. +The exact details are not too important, but in this piece of code, we happen to be +converting from the `rustc_hir::Ty` to a real `ty::Ty`. You can see that we first get some args +(`args`). Then we call `type_of` to get a type and call `ty.instantiate(tcx, args)` to get a new +version of `ty` with the args made. -[substex]: https://github.com/rust-lang/rust/blob/0940040c0486a536be4f8685c7dd9a078f9e87c2/compiler/rustc_hir_analysis/src/astconv/mod.rs#L1231-L1242 +[`instantiate`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/generic_args/struct.EarlyBinder.html#method.instantiate +[instantiatex]: https://github.com/rust-lang/rust/blob/8a562f9671e36cf29c9c794c2646bcf252d55535/compiler/rustc_hir_analysis/src/astconv/mod.rs#L905-L927 **Note on indices:** It is possible for the indices in `Param` to not match with what we expect. For example, the index could be out of bounds or it could be the index of a lifetime when we were expecting a type. These sorts of errors would be caught earlier in the compiler when translating from a `rustc_hir::Ty` to a `ty::Ty`. If they occur later, that is a compiler bug. - - diff --git a/src/getting-started.md b/src/getting-started.md index 4e1f520ff..886a91801 100644 --- a/src/getting-started.md +++ b/src/getting-started.md @@ -1,13 +1,37 @@ # Getting Started +Thank you for your interest in contributing to Rust! There are many ways to +contribute, and we appreciate all of them. + +If this is your first time contributing, the [walkthrough] chapter can give you a good example of +how a typical contribution would go. + This documentation is _not_ intended to be comprehensive; it is meant to be a quick guide for the most useful things. For more information, [see this chapter on how to build and run the compiler](./building/how-to-build-and-run.md). +[internals]: https://internals.rust-lang.org +[rust-discord]: http://discord.gg/rust-lang +[rust-zulip]: https://rust-lang.zulipchat.com +[coc]: https://www.rust-lang.org/conduct.html +[walkthrough]: ./walkthrough.md +[Getting Started]: ./getting-started.md + ## Asking Questions +If you have questions, please make a post on the [Rust Zulip server][rust-zulip] or +[internals.rust-lang.org][internals]. If you are contributing to Rustup, be aware they are not on +Zulip - you can ask questions in `#wg-rustup` [on Discord][rust-discord]. +See the [list of teams and working groups][governance] and [the Community page][community] on the +official website for more resources. + +[governance]: https://www.rust-lang.org/governance +[community]: https://www.rust-lang.org/community + +As a reminder, all contributors are expected to follow our [Code of Conduct][coc]. + The compiler team (or `t-compiler`) usually hangs out in Zulip [in this "stream"][z]; it will be easiest to get questions answered there. @@ -24,12 +48,18 @@ guide :) ### Experts Not all `t-compiler` members are experts on all parts of `rustc`; it's a pretty -large project. To find out who has expertise on different parts of the +large project. To find out who has expertise on different parts of the compiler, [consult this "experts map"][map]. -It's not perfectly complete, though, so please also feel free to ask questions +The experts map incomplete and not actively maintained, so please also feel free to ask questions even if you can't figure out who to ping. +Another way to find experts for a given part of the compiler is to see who has made recent commits. +For example, to find people who have recently worked on name resolution since the 1.68.2 release, +you could run `git shortlog -n 1.68.2.. compiler/rustc_resolve/`. Ignore any commits starting with +"Rollup merge" or commits by `@bors` (see [CI contribution procedures](./contributing.md#ci) for +more information about these commits). + [map]: https://github.com/rust-lang/compiler-team/blob/master/content/experts/map.toml ### Etiquette @@ -42,87 +72,65 @@ Just pinging someone without providing any context can be a bit annoying and just create noise, so we ask that you be mindful of the fact that the `t-compiler` folks get a lot of pings in a day. -## Cloning and Building - -### System Requirements - -Internet access is required. - -The most notable software requirement is that you will need Python 2 or 3, but -there are various others. - -The following hardware is recommended. -* 30GB+ of free disk space. -* 8GB+ RAM -* 2+ cores - -More powerful machines will lead to much faster builds. There are various -strategies to work around lesser hardware in the following chapters. - -See [this chapter][prereqs] for more details about software and hardware prerequisites. +## What should I work on? -[prereqs]: ./building/prerequisites.md +The Rust project is quite large and it can be difficult to know which parts of the project need +help, or are a good starting place for beginners. Here are some suggested starting places. -### Cloning +### Easy or mentored issues -You can just do a normal git clone: +If you're looking for somewhere to start, check out the following [issue +search][help-wanted-search]. See the [Triage] for an explanation of these labels. You can also try +filtering the search to areas you're interested in. For example: -```sh -git clone https://github.com/rust-lang/rust.git -cd rust -``` +- `repo:rust-lang/rust-clippy` will only show clippy issues +- `label:T-compiler` will only show issues related to the compiler +- `label:A-diagnostics` will only show diagnostic issues -### `x.py` Intro +Not all important or beginner work has issue labels. +See below for how to find work that isn't labelled. -`rustc` is a [bootstrapping] compiler, which makes it more complex than a -typical Rust program. As a result, you cannot use Cargo to build it. Instead -you must use the special tool `x.py`. It is used for the things Cargo is -normally used for: building, testing, creating releases, formatting, etc. +[help-wanted-search]: https://github.com/issues?q=is%3Aopen+is%3Aissue+org%3Arust-lang+no%3Aassignee+label%3AE-easy%2C%22good+first+issue%22%2Cgood-first-issue%2CE-medium%2CEasy%2CE-help-wanted%2CE-mentor+-label%3AS-blocked+ +[Triage]: ./contributing.md#issue-triage -[bootstrapping]: ./building/bootstrapping.md +### Recurring work -### Configuring the Compiler +Some work is too large to be done by a single person. In this case, it's common to have "Tracking +issues" to co-ordinate the work between contributors. Here are some example tracking issues where +it's easy to pick up work without a large time commitment: -In the top level of the repo: +- [Rustdoc Askama Migration](https://github.com/rust-lang/rust/issues/108868) +- [Diagnostic Translation](https://github.com/rust-lang/rust/issues/100717) +- [Move UI tests to subdirectories](https://github.com/rust-lang/rust/issues/73494) -```sh -$ ./x.py setup -``` +If you find more recurring work, please feel free to add it here! -This will do some initialization and walk you through an interactive setup to -create `config.toml`, the primary configuration file. +### Clippy issues -See [this chapter][config] for more info about configuration. +The [Clippy] project has spent a long time making its contribution process as friendly to newcomers +as possible. Consider working on it first to get familiar with the process and the compiler +internals. -[config]: ./building/how-to-build-and-run.md#create-a-configtoml +See [the Clippy contribution guide][clippy-contributing] for instructions on getting started. -### Common `x.py` commands +[Clippy]: https://doc.rust-lang.org/clippy/ +[clippy-contributing]: https://github.com/rust-lang/rust-clippy/blob/master/CONTRIBUTING.md -Here are the basic invocations of the `x.py` commands most commonly used when -working on `rustc`, `std`, `rustdoc`, and other tools. +### Diagnostic issues -| Command | When to use it | -| --- | --- | -| `./x.py check` | Quick check to see if most things compile; [rust-analyzer can run this automatically for you][rust-analyzer] | -| `./x.py build` | Builds `rustc`, `std`, and `rustdoc` | -| `./x.py test` | Runs all tests | -| `./x.py fmt` | Formats all code | +Many diagnostic issues are self-contained and don't need detailed background knowledge of the +compiler. You can see a list of diagnostic issues [here][diagnostic-issues]. -As written, these commands are reasonable starting points. However, there are -additional options and arguments for each of them that are worth learning for -serious development work. In particular, `./x.py build` and `./x.py test` -provide many ways to compile or test a subset of the code, which can save a lot -of time. +[diagnostic-issues]: https://github.com/rust-lang/rust/issues?q=is%3Aissue+is%3Aopen+label%3AA-diagnostics+no%3Aassignee -[rust-analyzer]: ./building/suggested.html#configuring-rust-analyzer-for-rustc +### Contributing to std (standard library) -See the chapters on [building](./building/how-to-build-and-run.md), -[testing](./tests/intro.md), and [rustdoc](./rustdoc.md) for more details. +See [std-dev-guide](https://std-dev-guide.rust-lang.org/). ### Contributing code to other Rust projects There are a bunch of other projects that you can contribute to outside of the -`rust-lang/rust` repo, including `clippy`, `miri`, `chalk`, and many others. +`rust-lang/rust` repo, including `cargo`, `miri`, `rustup`, and many others. These repos might have their own contributing guidelines and procedures. Many of them are owned by working groups (e.g. `chalk` is largely owned by @@ -142,173 +150,36 @@ incredibly helpful: - [Writing documentation][wd]: if you are feeling a bit more intrepid, you could try to read a part of the code and write doc comments for it. This will help you to learn some part of the compiler while also producing a useful artifact! +- [Triaging issues][triage]: categorizing, replicating, and minimizing issues is very helpful to the Rust maintainers. - [Working groups][wg]: there are a bunch of working groups on a wide variety of rust-related things. - +- Answer questions in the _Get Help!_ channels on the [Rust Discord + server][rust-discord], on [users.rust-lang.org][users], or on + [StackOverflow][so]. +- Participate in the [RFC process](https://github.com/rust-lang/rfcs). +- Find a [requested community library][community-library], build it, and publish + it to [Crates.io](http://crates.io). Easier said than done, but very, very + valuable! + +[rust-discord]: https://discord.gg/rust-lang +[users]: https://users.rust-lang.org/ +[so]: http://stackoverflow.com/questions/tagged/rust +[community-library]: https://github.com/rust-lang/rfcs/labels/A-community-library [iceb]: ./notification-groups/cleanup-crew.md [wd]: ./contributing.md#writing-documentation [wg]: https://rust-lang.github.io/compiler-team/working-groups/ +[triage]: ./contributing.md#issue-triage -## Contributor Procedures - -There are some official procedures to know about. This is a tour of the -highlights, but there are a lot more details, which we will link to below. - -### Code Review - -When you open a PR on the `rust-lang/rust` repo, a bot called `@rustbot` will -automatically assign a reviewer to the PR based on which files you changed. -The reviewer is the person that will approve the PR to be tested and merged. -If you want a specific reviewer (e.g. a team member you've been working with), -you can specifically request them by writing `r? @user` (e.g. `r? @jyn514`) in -either the original post or a followup comment -(you can see [this comment][r?] for example). - -Please note that the reviewers are humans, who for the most part work on `rustc` -in their free time. This means that they can take some time to respond and review -your PR. It also means that reviewers can miss some PRs that are assigned to them. - -To try to move PRs forward, the Triage WG regularly goes through all PRs that -are waiting for review and haven't been discussed for at least 2 weeks. If you -don't get a review within 2 weeks, feel free to ask the Triage WG on -Zulip ([#t-release/triage]). They have knowledge of when to ping, who might be -on vacation, etc. - -The reviewer may request some changes using the GitHub code review interface. -They may also request special procedures (such as a [crater] run; [see -below][break]) for some PRs. - -[r?]: https://github.com/rust-lang/rust/pull/78133#issuecomment-712692371 -[#t-release/triage]: https://rust-lang.zulipchat.com/#narrow/stream/242269-t-release.2Ftriage -[break]: #breaking-changes - -When the PR is ready to be merged, the reviewer will issue a command to -`@bors`, the CI bot. Usually, this is `@bors r+` or `@bors r=user` to approve -a PR (there are few other commands, but they are less relevant here). -You can see [this comment][r+] for example. This puts the PR in [bors's queue][bors] -to be tested and merged. Be patient; this can take a while and the queue can -sometimes be long. PRs are never merged by hand. - -[r+]: https://github.com/rust-lang/rust/pull/78133#issuecomment-712726339 -[bors]: https://bors.rust-lang.org/queue/rust - -### Bug Fixes or "Normal" code changes - -For most PRs, no special procedures are needed. You can just open a PR, and it -will be reviewed, approved, and merged. This includes most bug fixes, -refactorings, and other user-invisible changes. The next few sections talk -about exceptions to this rule. - -Also, note that it is perfectly acceptable to open WIP PRs or GitHub [Draft -PRs][draft]. Some people prefer to do this so they can get feedback along the -way or share their code with a collaborator. Others do this so they can utilize -the CI to build and test their PR (e.g. if you are developing on a laptop). - -[draft]: https://github.blog/2019-02-14-introducing-draft-pull-requests/ - -### New Features - -Rust has strong backwards-compatibility guarantees. Thus, new features can't -just be implemented directly in stable Rust. Instead, we have 3 release -channels: stable, beta, and nightly. - -- **Stable**: this is the latest stable release for general usage. -- **Beta**: this is the next release (will be stable within 6 weeks). -- **Nightly**: follows the `master` branch of the repo. This is the only - channel where unstable, incomplete, or experimental features are usable with - feature gates. - -In order to implement a new feature, usually you will need to go through [the -RFC process][rfc] to propose a design, have discussions, etc. In some cases, -small features can be added with only an FCP ([see below][break]). If in doubt, ask the -compiler, language, or libs team (whichever is most relevant). - -[rfc]: https://github.com/rust-lang/rfcs/blob/master/README.md - -After a feature is approved to be added, a tracking issue is created on the -`rust-lang/rust` repo, which tracks the progress towards the implementation of -the feature, any bugs reported, and eventually stabilization. - -The feature then needs to be implemented behind a feature gate, which prevents -it from being accidentally used. - -Finally, somebody may propose stabilizing the feature in an upcoming version of -Rust. This requires a Final Comment Period ([see below][break]) to get the -approval of the relevant teams. - -After that, the feature gate can be removed and the feature turned on for all users. - -For more details on this process, see [this chapter on implementing new -features.](./implementing_new_features.md) - -### Breaking Changes - -As mentioned above, Rust has strong backwards-compatibility guarantees. To this -end, we are reluctant to make breaking changes. However, sometimes they are -needed to correct compiler bugs (e.g. code that compiled but should not) or -make progress on some features. - -Depending on the scale of the breakage, there are a few different actions that -can be taken. If the reviewer believes the breakage is very minimal (i.e. very -unlikely to be actually encountered by users), they may just merge the change. -More often, they will request a Final Comment Period (FCP), which calls for -rough consensus among the members of a relevant team. The team members can -discuss the issue and either accept, reject, or request changes on the PR. - -If the scale of breakage is large, a deprecation warning may be needed. This is -a warning that the compiler will display to users whose code will break in the -future. After some time, an FCP can be used to move forward with the actual -breakage. - -If the scale of breakage is unknown, a team member or contributor may request a -[crater] run. This is a bot that will compile all crates.io crates and many -public github repos with the compiler with your changes. A report will then be -generated with crates that ceased to compile with or began to compile with your -changes. Crater runs can take a few days to complete. - -[crater]: https://github.com/rust-lang/crater - -### Major Changes - -The compiler team has a special process for large changes, whether or not they -cause breakage. This process is called a Major Change Proposal (MCP). MCP is a -relatively lightweight mechanism for getting feedback on large changes to the -compiler (as opposed to a full RFC or a design meeting with the team). - -Example of things that might require MCPs include major refactorings, changes -to important types, or important changes to how the compiler does something, or -smaller user-facing changes. - -**When in doubt, ask on [zulip][z]. It would be a shame to put a lot of work -into a PR that ends up not getting merged!** [See this document][mcpinfo] for -more info on MCPs. - -[mcpinfo]: https://forge.rust-lang.org/compiler/mcp.html - -### Performance - -Compiler performance is important. We have put a lot of effort over the last -few years into [gradually improving it][perfdash]. - -[perfdash]: https://perf.rust-lang.org/dashboard.html +## Cloning and Building -If you suspect that your change may cause a performance regression (or -improvement), you can request a "perf run" (your reviewer may also request one -before approving). This is yet another bot that will compile a collection of -benchmarks on a compiler with your changes. The numbers are reported -[here][perf], and you can see a comparison of your changes against the latest -master. +See ["How to build and run the compiler"](./building/how-to-build-and-run.md). -For an introduction to the performance of Rust code in general -which would also be useful in rustc development, see [The Rust Performance Book]. +## Contributor Procedures -[perf]: https://perf.rust-lang.org -[The Rust Performance Book]: https://nnethercote.github.io/perf-book/ +This section has moved to the ["Contribution Procedures"](./contributing.md) chapter. ## Other Resources -- This guide: talks about how `rustc` works -- [The t-compiler zulip][z] -- [The compiler's documentation (rustdocs)](https://doc.rust-lang.org/nightly/nightly-rustc/) -- [The Forge](https://forge.rust-lang.org/) has more documentation about various procedures. -- `#contribute` and `#rustdoc` on [Discord](https://discord.gg/rust-lang). +This section has moved to the ["About this guide"][more-links] chapter. + +[more-links]: ./about-this-guide.md#other-places-to-find-information diff --git a/src/git.md b/src/git.md index 65397e349..4208305a9 100644 --- a/src/git.md +++ b/src/git.md @@ -10,12 +10,16 @@ can be incorporated into the compiler. The goal of this page is to cover some of the more common questions and problems new contributors face. Although some Git basics will be covered here, -if you find that this is still a little too fast for you, it might make sense +if you find that this is still a little too fast for you, it might make sense to first read some introductions to Git, such as the Beginner and Getting started sections of [this tutorial from Atlassian][atlassian-git]. GitHub also provides [documentation] and [guides] for beginners, or you can consult the more in depth [book from Git]. +This guide is incomplete. If you run into trouble with git that this page doesn't help with, +please [open an issue] so we can document how to fix it. + +[open an issue]: https://github.com/rust-lang/rustc-dev-guide/issues/new [book from Git]: https://git-scm.com/book/en/v2/ [atlassian-git]: https://www.atlassian.com/git/tutorials/what-is-version-control [documentation]: https://docs.github.com/en/get-started/quickstart/set-up-git @@ -58,7 +62,7 @@ and PRs: 1. Ensure that you're making your changes on top of master: `git checkout master`. 2. Get the latest changes from the Rust repo: `git pull upstream master --ff-only`. - (see [No-Merge Policy](#keeping-things-up-to-date) for more info about this). + (see [No-Merge Policy][no-merge-policy] for more info about this). 3. Make a new branch for your change: `git checkout -b issue-12345-fix`. 4. Make some changes to the repo and test them. 5. Stage your changes via `git add src/changed/file.rs src/another/change.rs` @@ -76,7 +80,7 @@ pulling-and-rebasing, you can use `git push --force-with-lease`). If you end up needing to rebase and are hitting conflicts, see [Rebasing](#rebasing). If you want to track upstream while working on long-running feature/issue, see -[Keeping things up to date](#keeping-things-up-to-date). +[Keeping things up to date][no-merge-policy]. If your reviewer requests changes, the procedure for those changes looks much the same, with some steps skipped: @@ -86,6 +90,8 @@ the same, with some steps skipped: 2. Make, stage, and commit your additional changes just like before. 3. Push those changes to your fork: `git push`. + [no-merge-policy]: #keeping-things-up-to-date + ## Troubleshooting git issues You don't need to clone `rust-lang/rust` from scratch if it's out of date! @@ -93,6 +99,14 @@ Even if you think you've messed it up beyond repair, there are ways to fix the git state that don't require downloading the whole repository again. Here are some common issues you might run into: +### I made a merge commit by accident. + +Git has two ways to update your branch with the newest changes: merging and rebasing. +Rust [uses rebasing][no-merge-policy]. If you make a merge commit, it's not too hard to fix: +`git rebase -i upstream/master`. + +See [Rebasing](#rebasing) for more about rebasing. + ### I deleted my fork on GitHub! This is not a problem from git's perspective. If you run `git remote -v`, @@ -100,20 +114,72 @@ it will say something like this: ``` $ git remote -v -origin https://github.com//rust-lang/rust (fetch) -origin https://github.com//rust-lang/rust (push) -personal https://github.com/jyn514/rust (fetch) -personal https://github.com/jyn514/rust (push) +origin git@github.com:jyn514/rust.git (fetch) +origin git@github.com:jyn514/rust.git (push) +upstream https://github.com/rust-lang/rust (fetch) +upstream https://github.com/rust-lang/rust (fetch) ``` If you renamed your fork, you can change the URL like this: ```console -git remote set-url personal +git remote set-url origin ``` where the `` is your new fork. +### I changed a submodule by accident + +Usually people notice this when rustbot posts a comment on github that `cargo` has been modified: + +![rustbot submodule comment](./img/rustbot-submodules.png) + +You might also notice conflicts in the web UI: + +![conflict in src/tools/cargo](./img/submodule-conflicts.png) + +The most common cause is that you rebased after a change and ran `git add .` without first running +`x` to update the submodules. Alternatively, you might have run `cargo fmt` instead of `x fmt` +and modified files in a submodule, then committed the changes. + +To fix it, do the following things: + +1. See which commit has the accidental changes: `git log --stat -n1 src/tools/cargo` +2. Revert the changes to that commit: `git checkout ~ src/tools/cargo`. Type `~` + literally but replace `` with the output from step 1. +3. Tell git to commit the changes: `git commit --fixup ` +4. Repeat steps 1-3 for all the submodules you modified. + - If you modified the submodule in several different commits, you will need to repeat steps 1-3 + for each commit you modified. You'll know when to stop when the `git log` command shows a commit + that's not authored by you. +5. Squash your changes into the existing commits: `git rebase --autosquash -i upstream/master` +6. [Push your changes](#standard-process). + +### I see "error: cannot rebase" when I try to rebase + +These are two common errors to see when rebasing: +``` +error: cannot rebase: Your index contains uncommitted changes. +error: Please commit or stash them. +``` +``` +error: cannot rebase: You have unstaged changes. +error: Please commit or stash them. +``` + +(See for the difference between the two.) + +This means you have made changes since the last time you made a commit. To be able to rebase, either +commit your changes, or make a temporary commit called a "stash" to have them still not be commited +when you finish rebasing. You may want to configure git to make this "stash" automatically, which +will prevent the "cannot rebase" error in nearly all cases: + +``` +git config --global rebase.autostash true +``` + +See for more info about stashing. + ### I see 'Untracked Files: src/stdarch'? This is left over from the move to the `library/` directory. @@ -135,6 +201,42 @@ and just want to get a clean copy of the repository back, you can use `git reset git reset --hard master ``` +### failed to push some refs + +`git push` will not work properly and say something like this: + +``` + ! [rejected] issue-xxxxx -> issue-xxxxx (non-fast-forward) +error: failed to push some refs to 'https://github.com/username/rust.git' +hint: Updates were rejected because the tip of your current branch is behind +hint: its remote counterpart. Integrate the remote changes (e.g. +hint: 'git pull ...') before pushing again. +hint: See the 'Note about fast-forwards' in 'git push --help' for details. +``` + +The advice this gives is incorrect! Because of Rust's +["no-merge" policy](#no-merge-policy) the merge commit created by `git pull` +will not be allowed in the final PR, in addition to defeating the point of the +rebase! Use `git push --force-with-lease` instead. + +### Git is trying to rebase commits I didn't write? + +If you see many commits in your rebase list, or merge commits, or commits by other people that you +didn't write, it likely means you're trying to rebase over the wrong branch. For example, you may +have a `rust-lang/rust` remote `upstream`, but ran `git rebase origin/master` instead of `git rebase +upstream/master`. The fix is to abort the rebase and use the correct branch instead: + +``` +git rebase --abort +git rebase -i upstream/master +``` + +
Click here to see an example of rebasing over the wrong branch + +![Interactive rebase over the wrong branch](img/other-peoples-commits.png) + +
+ ### Quick note about submodules When updating your local repository with `git pull`, you may notice that sometimes @@ -154,29 +256,13 @@ Changes not staged for commit: no changes added to commit (use "git add" and/or "git commit -a") ``` -These changes are not changes to files: they are changes to submodules (more on -this [later](#git-submodules)). To get rid of those, run `git submodule update` -(or run any `x.py` command, which will automatically update the submodules). -Note that, -as of Aug 2022, -there is a [bug][#77620] if you use worktrees, -submodules, and `x.py` in a commit hook. -If you run into an error like the following, -it's not anything you did wrong: - -``` -error: failed to read `/home/joshua/rustc-worktree/src/tools/miri/cargo-miri/Cargo.toml` - -Caused by: - No such file or directory (os error 2) -``` - -There is a workaround in [the issue][#77620-workaround]. +These changes are not changes to files: they are changes to submodules (more on this +[later](#git-submodules)). To get rid of those, run `./x --help`, which will automatically update +the submodules. -[#77620]: https://github.com/rust-lang/rust/issues/77620 -[#77620-workaround]: https://github.com/rust-lang/rust/issues/77620#issuecomment-705228229 - -(Note that as of Sept 2022 `miri` is a subtree and not a submodule.) +Some submodules are not actually needed; for example, `src/llvm-project` doesn't need to be checked +out if you're using `download-ci-llvm`. To avoid having to keep fetching its history, you can use +`git submodule deinit -f src/llvm-project`, which will also avoid it showing as modified again. ## Rebasing and Conflicts @@ -228,7 +314,7 @@ Generally, resolving the conflict consists of two steps: First, fix the particular conflict. Edit the file to make the changes you want and remove the `<<<<<<<`, `=======` and `>>>>>>>` lines in the process. Second, check the surrounding code. If there was a conflict, its likely there are some logical -errors lying around too! It's a good idea to run `x.py check` here to make sure +errors lying around too! It's a good idea to run `x check` here to make sure there are no glaring errors. Once you're all done fixing the conflicts, you need to stage the files that had @@ -238,22 +324,6 @@ Git know that you've resolved the conflicts and it should finish the rebase. Once the rebase has succeeded, you'll want to update the associated branch on your fork with `git push --force-with-lease`. -Note that `git push` will not work properly and say something like this: - -``` - ! [rejected] issue-xxxxx -> issue-xxxxx (non-fast-forward) -error: failed to push some refs to 'https://github.com/username/rust.git' -hint: Updates were rejected because the tip of your current branch is behind -hint: its remote counterpart. Integrate the remote changes (e.g. -hint: 'git pull ...') before pushing again. -hint: See the 'Note about fast-forwards' in 'git push --help' for details. -``` - -The advice this gives is incorrect! Because of Rust's -["no-merge" policy](#no-merge-policy) the merge commit created by `git pull` -will not be allowed in the final PR, in addition to defeating the point of the -rebase! Use `git push --force-with-lease` instead. - ### Keeping things up to date The above section on [Rebasing](#rebasing) is a specific @@ -269,9 +339,8 @@ and rebase them: ``` git checkout master git pull upstream master --ff-only # to make certain there are no merge commits -git checkout feature_branch -git rebase master -git push --force-with-lease (set origin to be the same as local) +git rebase master feature_branch +git push --force-with-lease # (set origin to be the same as local) ``` To avoid merges as per the [No-Merge Policy](#no-merge-policy), you may want to use @@ -279,11 +348,13 @@ To avoid merges as per the [No-Merge Policy](#no-merge-policy), you may want to to ensure that Git doesn't create merge commits when `git pull`ing, without needing to pass `--ff-only` or `--rebase` every time. -You can also `git push --force-with-lease` from master to keep your origin's master in sync with +You can also `git push --force-with-lease` from master to keep your fork's master in sync with upstream. ## Advanced Rebasing +### Squash your commits + If your branch contains multiple consecutive rewrites of the same code, or if the rebase conflicts are extremely severe, you can use `git rebase --interactive master` to gain more control over the process. This @@ -376,7 +447,7 @@ that merge commits in PRs are not accepted. As a result, if you are running course, this is not always true; if your merge will just be a fast-forward, like the merges that `git pull` usually performs, then no merge commit is created and you have nothing to worry about. Running `git config merge.ff only` -(this will apply the config to the local repo). +(this will apply the config to the local repo) once will ensure that all the merges you perform are of this type, so that you cannot make a mistake. @@ -385,6 +456,56 @@ tradeoff. The main advantage is the generally linear commit history. This greatly simplifies bisecting and makes the history and commit log much easier to follow and understand. +## Tips for reviewing + +**NOTE**: This section is for *reviewing* PRs, not authoring them. + +### Hiding whitespace + +Github has a button for disabling whitespace changes that may be useful. +You can also use `git diff -w origin/master` to view changes locally. + +![hide whitespace](./img/github-whitespace-changes.png) + +### Fetching PRs + +To checkout PRs locally, you can use `git fetch upstream pull/NNNNN/head && git checkout +FETCH_HEAD`. + +You can also use github's cli tool. Github shows a button on PRs where you can copy-paste the +command to check it out locally. See for more info. + +![`gh` suggestion](./img/github-cli.png) + +### Moving large sections of code + +Git and Github's default diff view for large moves *within* a file is quite poor; it will show each +line as deleted and each line as added, forcing you to compare each line yourself. Git has an option +to show moved lines in a different color: + +``` +git log -p --color-moved=dimmed-zebra --color-moved-ws=allow-indentation-change +``` + +See [the docs for `--color-moved`](https://git-scm.com/docs/git-diff#Documentation/git-diff.txt---color-movedltmodegt) for more info. + +### range-diff + +See [the relevant section for PR authors](#git-range-diff). This can be useful for comparing code +that was force-pushed to make sure there are no unexpected changes. + +### Ignoring changes to specific files + +Many large files in the repo are autogenerated. To view a diff that ignores changes to those files, +you can use the following syntax (e.g. Cargo.lock): + +``` +git log -p ':!Cargo.lock' +``` + +Arbitrary patterns are supported (e.g. `:!compiler/*`). Patterns use the same syntax as +`.gitignore`, with `:` prepended to indicate a pattern. + ## Git submodules **NOTE**: submodules are a nice thing to know about, but it *isn't* an absolute @@ -423,7 +544,7 @@ commit, which doesn't change unless modified manually. If you use `git checkout in the `llvm-project` directory and go back to the `rust` directory, you can stage this change like any other, e.g. by running `git add src/llvm-project`. (Note that if you *don't* stage the change to commit, then you run the risk that running -`x.py` will just undo your change by switching back to the previous commit when +`x` will just undo your change by switching back to the previous commit when it automatically "updates" the submodules.) This version selection is usually done by the maintainers of the project, and diff --git a/src/hir-debugging.md b/src/hir-debugging.md index 8788ad4de..5a0bda208 100644 --- a/src/hir-debugging.md +++ b/src/hir-debugging.md @@ -1,8 +1,15 @@ # HIR Debugging -The `-Z unpretty=hir-tree` flag will dump out the HIR. +Use the `-Z unpretty=hir` flag to produce a human-readable representation of the HIR. +For cargo projects this can be done with `cargo rustc -- -Z unpretty=hir`. +This output is useful when you need to see at a glance how your code was desugared and transformed +during AST lowering. + +For a full `Debug` dump of the data in the HIR, use the `-Z unpretty=hir-tree` flag. +This may be useful when you need to see the full structure of the HIR from the perspective of the +compiler. If you are trying to correlate `NodeId`s or `DefId`s with source code, the -`--pretty expanded,identified` flag may be useful. +`-Z unpretty=expanded,identified` flag may be useful. TODO: anything else? [#1159](https://github.com/rust-lang/rustc-dev-guide/issues/1159) diff --git a/src/hir.md b/src/hir.md index 8316f7ce3..160b47645 100644 --- a/src/hir.md +++ b/src/hir.md @@ -117,9 +117,9 @@ that `n` must be some HIR expression, you can do [Expr]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/struct.Expr.html Finally, you can use the HIR map to find the parents of nodes, via -calls like [`tcx.hir().get_parent_node(n)`][get_parent_node]. +calls like [`tcx.hir().get_parent(n)`][get_parent]. -[get_parent_node]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/map/struct.Map.html#method.get_parent_node +[get_parent]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/hir/map/struct.Map.html#method.get_parent ## HIR Bodies diff --git a/src/identifiers.md b/src/identifiers.md index 18a3efefc..c7e32d04c 100644 --- a/src/identifiers.md +++ b/src/identifiers.md @@ -1,4 +1,4 @@ -# Identifiers in the Compiler +# Identifiers in the compiler If you have read the few previous chapters, you now know that `rustc` uses many different intermediate representations to perform different kinds of analyses. @@ -65,13 +65,12 @@ See the [HIR chapter][hir-map] for more detailed information. - [`BasicBlock`] identifies a *basic block*. It points to an instance of [`BasicBlockData`], which can be retrieved by indexing into - [`Body::basic_blocks()`] (note that you must call a function; the field is - private). + [`Body.basic_blocks`]. - [`Local`] identifies a local variable in a function. Its associated data is in [`LocalDecl`], which can be retrieved by indexing into [`Body.local_decls`]. -- [`Field`] identifies a struct's, union's, or enum variant's field. It is used +- [`FieldIdx`] identifies a struct's, union's, or enum variant's field. It is used as a "projection" in [`Place`]. - [`SourceScope`] identifies a name scope in the original source code. Used for @@ -93,11 +92,11 @@ See the [HIR chapter][hir-map] for more detailed information. [`BasicBlock`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.BasicBlock.html [`BasicBlockData`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.BasicBlockData.html -[`Body::basic_blocks()`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Body.html#method.basic_blocks +[`Body.basic_blocks`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Body.html#structfield.basic_blocks [`Local`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Local.html [`LocalDecl`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.LocalDecl.html [`Body.local_decls`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Body.html#structfield.local_decls -[`Field`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Field.html +[`FieldIdx`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_abi/struct.FieldIdx.html [`Place`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Place.html [`SourceScope`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.SourceScope.html [`SourceScopeData`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.SourceScopeData.html diff --git a/src/img/coverage-graphviz-01.png b/src/img/coverage-graphviz-01.png deleted file mode 100644 index cf7e1e76b..000000000 Binary files a/src/img/coverage-graphviz-01.png and /dev/null differ diff --git a/src/img/coverage-spanview-01.png b/src/img/coverage-spanview-01.png deleted file mode 100644 index f3e4b3535..000000000 Binary files a/src/img/coverage-spanview-01.png and /dev/null differ diff --git a/src/img/github-cli.png b/src/img/github-cli.png new file mode 100644 index 000000000..c3b0e7707 Binary files /dev/null and b/src/img/github-cli.png differ diff --git a/src/img/github-whitespace-changes.png b/src/img/github-whitespace-changes.png new file mode 100644 index 000000000..9a19a10aa Binary files /dev/null and b/src/img/github-whitespace-changes.png differ diff --git a/src/img/other-peoples-commits.png b/src/img/other-peoples-commits.png new file mode 100644 index 000000000..e4fc2c797 Binary files /dev/null and b/src/img/other-peoples-commits.png differ diff --git a/src/img/rustbot-submodules.png b/src/img/rustbot-submodules.png new file mode 100644 index 000000000..c2e6937cb Binary files /dev/null and b/src/img/rustbot-submodules.png differ diff --git a/src/img/submodule-conflicts.png b/src/img/submodule-conflicts.png new file mode 100644 index 000000000..e90a6bbe8 Binary files /dev/null and b/src/img/submodule-conflicts.png differ diff --git a/src/implementing_new_features.md b/src/implementing_new_features.md index 9147c1b41..427589dab 100644 --- a/src/implementing_new_features.md +++ b/src/implementing_new_features.md @@ -1,9 +1,16 @@ -# Implementing new features +# Implementing new language features + + When you want to implement a new significant feature in the compiler, you need to go through this process to make sure everything goes smoothly. +**NOTE: this section is for *language* features, not *library* features, +which use [a different process].** + +[a different process]: ./stability.md + ## The @rfcbot FCP process When the change is small and uncontroversial, then it can be done @@ -33,7 +40,7 @@ like this; for example, the compiler team recommends filing a Major Change Proposal ([MCP][mcp]) as a lightweight way to garner support and feedback without requiring full consensus. -[mcp]: compiler/mcp.md#public-facing-changes-require-rfcbot-fcp +[mcp]: https://forge.rust-lang.org/compiler/mcp.html#public-facing-changes-require-rfcbot-fcp You don't need to have the implementation fully ready for r+ to propose an FCP, but it is generally a good idea to have at least a proof @@ -89,31 +96,16 @@ by being unstable and unchanged for a year. To keep track of the status of an unstable feature, the experience we get while using it on nightly, and of the concerns that block its stabilization, every feature-gate -needs a tracking issue. - -General discussions about the feature should be done on -the tracking issue. +needs a tracking issue. General discussions about the feature should be done on the tracking issue. For features that have an RFC, you should use the RFC's tracking issue for the feature. For other features, you'll have to make a tracking issue for that feature. The issue title should be "Tracking issue -for YOUR FEATURE". +for YOUR FEATURE". Use the ["Tracking Issue" issue template][template]. -For tracking issues for features (as opposed to future-compat -warnings), I don't think the description has to contain -anything specific. Generally we put the list of items required -for stabilization in a checklist, e.g., - -```txt -**Steps:** - -- [ ] Implement the RFC. (CC @rust-lang/compiler -- can anyone write - up mentoring instructions?) -- [ ] Adjust the documentation. ([See instructions on rustc-dev-guide.](stabilization_guide.md#documentation-prs)) -- [ ] Stabilize the feature. ([See instructions on rustc-dev-guide.](stabilization_guide.md#stabilization-pr)) -``` +[template]: https://github.com/rust-lang/rust/issues/new?template=tracking_issue.md ## Stability in code @@ -126,14 +118,48 @@ a new unstable feature: The tracking issue should be labeled with at least `C-tracking-issue`. For a language feature, a label `F-feature_name` should be added as well. -2. Pick a name for the feature gate (for RFCs, use the name +1. Pick a name for the feature gate (for RFCs, use the name in the RFC). -3. Add a feature gate declaration to `rustc_feature/src/active.rs` in the active - `declare_features` block, and add the feature gate keyword to - `rustc_span/src/symbol.rs`. See [here][add-feature-gate] for detailed instructions. +1. Add the feature name to `rustc_span/src/symbol.rs` in the `Symbols {...}` block. + +1. Add a feature gate declaration to `rustc_feature/src/unstable.rs` in the unstable + `declare_features` block. + + ```rust ignore + /// description of feature + (unstable, $feature_name, "CURRENT_RUSTC_VERSION", Some($tracking_issue_number), $edition) + ``` + + where `$edition` has the type `Option`, and is typically just `None`. If you haven't yet + opened a tracking issue (e.g. because you want initial feedback on whether the feature is likely + to be accepted), you can temporarily use `None` - but make sure to update it before the PR is + merged! + + For example: + + ```rust ignore + /// Allows defining identifiers beyond ASCII. + (unstable, non_ascii_idents, "CURRENT_RUSTC_VERSION", Some(55467), None), + ``` + + Features can be marked as incomplete, and trigger the warn-by-default [`incomplete_features` + lint] + by setting their type to `incomplete`: + + [`incomplete_features` lint]: https://doc.rust-lang.org/rustc/lints/listing/warn-by-default.html#incomplete-features + + ```rust ignore + /// Allows unsized rvalues at arguments and parameters. + (incomplete, unsized_locals, "CURRENT_RUSTC_VERSION", Some(48055), None), + ``` + + To avoid [semantic merge conflicts], please use `CURRENT_RUSTC_VERSION` instead of `1.70` or + another explicit version number. + + [semantic merge conflicts]: https://bors.tech/essay/2017/02/02/pitch/ -4. Prevent usage of the new feature unless the feature gate is set. +1. Prevent usage of the new feature unless the feature gate is set. You can check it in most places in the compiler using the expression `tcx.features().$feature_name` (or `sess.features_untracked().$feature_name` if the @@ -149,18 +175,18 @@ a new unstable feature: and then finally feature-gate all the spans in [`rustc_ast_passes::feature_gate::check_crate`]. -5. Add a test to ensure the feature cannot be used without - a feature gate, by creating `feature-gate-$feature_name.rs` - and `feature-gate-$feature_name.stderr` files under the - directory where the other tests for your feature reside. +1. Add a test to ensure the feature cannot be used without + a feature gate, by creating `tests/ui/feature-gates/feature-gate-$feature_name.rs`. + You can generate the corresponding `.stderr` file by running `./x test +tests/ui/feature-gates/ --bless`. -6. Add a section to the unstable book, in +1. Add a section to the unstable book, in `src/doc/unstable-book/src/language-features/$feature_name.md`. -7. Write a lot of tests for the new feature. +1. Write a lot of tests for the new feature, preferably in `tests/ui/$feature_name/`. PRs without tests will not be accepted! -8. Get your PR reviewed and land it. You have now successfully +1. Get your PR reviewed and land it. You have now successfully implemented a feature in Rust! [`GatedSpans`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_session/parse/struct.GatedSpans.html @@ -170,5 +196,5 @@ a new unstable feature: [value the stability of Rust]: https://github.com/rust-lang/rfcs/blob/master/text/1122-language-semver.md [stability in code]: #stability-in-code [here]: ./stabilization_guide.md -[tracking issue]: #tracking-issue +[tracking issue]: #tracking-issues [add-feature-gate]: ./feature-gates.md#adding-a-feature-gate diff --git a/src/incrcomp-debugging.md b/src/incrcomp-debugging.md index ba312c959..3606943dd 100644 --- a/src/incrcomp-debugging.md +++ b/src/incrcomp-debugging.md @@ -6,7 +6,7 @@ There are various ways to write tests against the dependency graph. The simplest mechanisms are the `#[rustc_if_this_changed]` and `#[rustc_then_this_would_need]` annotations. These are used in ui tests to test whether the expected set of paths exist in the dependency graph. -As an example, see `src/test/ui/dep-graph/dep-graph-caller-callee.rs`. +As an example, see `tests/ui/dep-graph/dep-graph-caller-callee.rs`. The idea is that you can annotate a test like: diff --git a/src/licenses.md b/src/licenses.md index 06cd8ae6b..aa8286714 100644 --- a/src/licenses.md +++ b/src/licenses.md @@ -3,3 +3,45 @@ The `rustc` compiler source and standard library are dual licensed under the [Apache License v2.0](https://github.com/rust-lang/rust/blob/master/LICENSE-APACHE) and the [MIT License](https://github.com/rust-lang/rust/blob/master/LICENSE-MIT) unless otherwise specified. Detailed licensing information is available in the [COPYRIGHT document](https://github.com/rust-lang/rust/blob/master/COPYRIGHT) of the `rust-lang/rust` repository. + +## Guidelines for reviewers + +In general, reviewers need to be looking not only for the code quality of contributions but also +that they are properly licensed. +We have some tips below for things to look out for when reviewing, but if you ever feel uncertain +as to whether some code might be properly licensed, err on the safe side — reach out to the Council +or Compiler Team Leads for feedback! + +Things to watch out for: + +- The PR author states that they copied, ported, or adapted the code from some other source. +- There is a comment in the code pointing to a webpage or describing where the algorithm was taken +from. +- The algorithm or code pattern seems like it was likely copied from somewhere else. +- When adding new dependencies, double check the dependency's license. + +In all of these cases, we will want to check that source to make sure it it is licensed in a way +that is compatible with Rust’s license. + +Examples + +- Porting C code from a GPL project, like GNU binutils, is not allowed. That would require Rust +itself to be licensed under the GPL. +- Copying code from an algorithms text book may be allowed, but some algorithms are patented. + +## Porting + +Contributions to rustc, especially around platform and compiler intrinsics, often include porting +over work from other projects, mainly LLVM and GCC. + +Some general rules apply: + +- Copying work needs to adhere to the original license + - This applies to direct copy & paste + - This also applies to code you looked at and ported + +In general, taking inspiration from other codebases is fine, but please exercise caution when +porting code. + +Ports of full libraries (e.g. C libraries shipped with LLVM) must keep the license of the original +library. diff --git a/src/llvm-coverage-instrumentation.md b/src/llvm-coverage-instrumentation.md index b186f4820..a21254946 100644 --- a/src/llvm-coverage-instrumentation.md +++ b/src/llvm-coverage-instrumentation.md @@ -28,7 +28,7 @@ them), and generate various reports for analysis, for example:
Detailed instructions and examples are documented in the -[Rustc Book][rustc-book-instrument-coverage]. +[rustc book][rustc-book-instrument-coverage]. [llvm-instrprof-increment]: https://llvm.org/docs/LangRef.html#llvm-instrprof-increment-intrinsic [coverage map]: https://llvm.org/docs/CoverageMappingFormat.html @@ -56,14 +56,14 @@ statically links coverage-instrumented binaries with LLVM runtime code ([compiler-rt][compiler-rt-profile]) that implements program hooks (such as an `exit` hook) to write the counter values to the `.profraw` file. -In the `rustc` source tree, `library/profiler_builtins` bundles the LLVM -`compiler-rt` code into a Rust library crate. (When building `rustc`, the -`profiler_builtins` library is only included when `profiler = true` is set -in `rustc`'s `config.toml`.) +In the `rustc` source tree, +`library/profiler_builtins` bundles the LLVM `compiler-rt` code into a Rust library crate. +Note that when building `rustc`, +`profiler_builtins` is only included when `build.profiler = true` is set in `config.toml`. When compiling with `-C instrument-coverage`, -[`CrateLoader::postprocess()`][crate-loader-postprocess] dynamically loads the -`profiler_builtins` library by calling `inject_profiler_runtime()`. +[`CrateLoader::postprocess()`][crate-loader-postprocess] dynamically loads +`profiler_builtins` by calling `inject_profiler_runtime()`. [compiler-rt-profile]: https://github.com/llvm/llvm-project/tree/main/compiler-rt/lib/profile [crate-loader-postprocess]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/creader/struct.CrateLoader.html#method.postprocess @@ -73,21 +73,21 @@ When compiling with `-C instrument-coverage`, Coverage instrumentation is performed on the MIR with a [MIR pass][mir-passes] called [`InstrumentCoverage`][mir-instrument-coverage]. This MIR pass analyzes the control flow graph (CFG)--represented by MIR `BasicBlock`s--to identify -code branches, and injects additional [`Coverage`][coverage-statement] -statements into the `BasicBlock`s. +code branches, attaches [`FunctionCoverageInfo`] to the function's body, +and injects additional [`Coverage`][coverage-statement] statements into the +`BasicBlock`s. A MIR `Coverage` statement is a virtual instruction that indicates a counter should be incremented when its adjacent statements are executed, to count a span of code ([`CodeRegion`][code-region]). It counts the number of times a -branch is executed, and also specifies the exact location of that code span in -the Rust source code. +branch is executed, and is referred to by coverage mappings in the function's +coverage-info struct. -Note that many of these `Coverage` statements will _not_ be converted into +Note that many coverage counters will _not_ be converted into physical counters (or any other executable instructions) in the final binary. -Some of them will be (see `CoverageKind::`[`Counter`][counter-coverage-kind]), +Some of them will be (see [`CoverageKind::CounterIncrement`]), but other counters can be computed on the fly, when generating a coverage -report, by mapping a `CodeRegion` to a -`CoverageKind`::[`Expression`][expression-coverage-kind]. +report, by mapping a `CodeRegion` to a coverage-counter _expression_. As an example: @@ -121,8 +121,8 @@ determines when to break out of a loop (a `while` condition, or an `if` or `match` with a `break`). In MIR, this is typically lowered to a `SwitchInt`, with one branch to stay in the loop, and another branch to break out of the loop. The branch that breaks out will almost always execute less often, -so `InstrumentCoverage` chooses to add a `Counter` to that branch, and an -`Expression(continue) = Counter(loop) - Counter(break)` to the branch that +so `InstrumentCoverage` chooses to add a `CounterIncrement` to that branch, and +uses an expression (`Counter(loop) - Counter(break)`) for the branch that continues. The `InstrumentCoverage` MIR pass is documented in @@ -130,9 +130,9 @@ The `InstrumentCoverage` MIR pass is documented in [mir-passes]: mir/passes.md [mir-instrument-coverage]: https://github.com/rust-lang/rust/tree/master/compiler/rustc_mir_transform/src/coverage +[`FunctionCoverageInfo`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/coverage/struct.FunctionCoverageInfo.html [code-region]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/coverage/struct.CodeRegion.html -[counter-coverage-kind]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/coverage/enum.CoverageKind.html#variant.Counter -[expression-coverage-kind]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/coverage/enum.CoverageKind.html#variant.Expression +[`CoverageKind::CounterIncrement`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/coverage/enum.CoverageKind.html#variant.CounterIncrement [coverage-statement]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.StatementKind.html#variant.Coverage [instrument-coverage-pass-details]: #implementation-details-of-the-instrumentcoverage-mir-pass @@ -150,40 +150,38 @@ MIR `Statement` into some backend-specific action or instruction. match statement.kind { ... mir::StatementKind::Coverage(box ref coverage) => { - self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope); - bx + self.codegen_coverage(bx, coverage, statement.source_info.scope); } ``` -`codegen_coverage()` handles each `CoverageKind` as follows: +`codegen_coverage()` handles inlined statements and then forwards the coverage +statement to [`Builder::add_coverage`], which handles each `CoverageKind` as +follows: -- For all `CoverageKind`s, Coverage data (counter ID, expression equation - and ID, and code regions) are passed to the backend's `Builder`, to - populate data structures that will be used to generate the crate's - "Coverage Map". (See the [`FunctionCoverage`][function-coverage] `struct`.) -- For `CoverageKind::Counter`s, an instruction is injected in the backend + +- For both `CounterIncrement` and `ExpressionUsed`, the underlying counter or + expression ID is passed through to the corresponding [`FunctionCoverage`] + struct to indicate that the corresponding regions of code were not removed + by MIR optimizations. +- For `CoverageKind::CounterIncrement`s, an instruction is injected in the backend IR to increment the physical counter, by calling the `BuilderMethod` [`instrprof_increment()`][instrprof-increment]. ```rust - pub fn codegen_coverage(&self, bx: &mut Bx, coverage: Coverage, scope: SourceScope) { + fn add_coverage(&mut self, instance: Instance<'tcx>, coverage: &Coverage) { ... - let instance = ... // the scoped instance (current or inlined function) - let Coverage { kind, code_region } = coverage; - match kind { - CoverageKind::Counter { function_source_hash, id } => { - ... - bx.add_coverage_counter(instance, id, code_region); + let Coverage { kind } = coverage; + match *kind { + CoverageKind::CounterIncrement { id } => { + func_coverage.mark_counter_id_seen(id); ... bx.instrprof_increment(fn_name, hash, num_counters, index); } - CoverageKind::Expression { id, lhs, op, rhs } => { - bx.add_coverage_counter_expression(instance, id, lhs, op, rhs, code_region); + CoverageKind::ExpressionUsed { id } => { + func_coverage.mark_expression_id_seen(id); } - CoverageKind::Unreachable => { - bx.add_coverage_unreachable( - instance, - code_region.expect(... + } + } ``` > The function name `instrprof_increment()` is taken from the LLVM intrinsic @@ -199,7 +197,8 @@ statements is only implemented for LLVM, at this time. [backend-lowering-mir]: backend/lowering-mir.md [codegen-statement]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/struct.FunctionCx.html#method.codegen_statement [codegen-coverage]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/struct.FunctionCx.html#method.codegen_coverage -[function-coverage]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/coverageinfo/map/struct.FunctionCoverage.html +[`Builder::add_coverage`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/builder/struct.Builder.html#method.add_coverage +[`FunctionCoverage`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/coverageinfo/map_data/struct.FunctionCoverage.html [instrprof-increment]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/traits/trait.BuilderMethods.html#tymethod.instrprof_increment ### Coverage Map Generation @@ -222,13 +221,9 @@ properly-configured variables in LLVM IR, according to very specific details of the [_LLVM Coverage Mapping Format_][coverage-mapping-format] (Version 6).[^llvm-and-covmap-versions] -[^llvm-and-covmap-versions]: The Rust compiler (as of December 2021) -supports _LLVM Coverage Mapping Format_ Version 5 or 6. Version 5 -was introduced in _LLVM 12_, -which is (as of this writing) the minimum LLVM -version supported by the current version of Rust. Version 6 was introduced in -_LLVM 13_, which is currently the default LLVM version for Rust. The Rust -compiler will automatically use the most up-to-date coverage mapping format +[^llvm-and-covmap-versions]: +The Rust compiler (as of Jul 2023) supports _LLVM Coverage Mapping Format_ 6. +The Rust compiler will automatically use the most up-to-date coverage mapping format version that is compatible with the compiler's built-in version of LLVM. ```rust @@ -278,34 +273,45 @@ since it will not be called), and adds a new `FunctionCoverage`, with ## Testing LLVM Coverage -Coverage instrumentation in the MIR is validated by a `mir-opt` test: -[`instrument-coverage`][mir-opt-test]. - -More complete testing of end-to-end coverage instrumentation and reports are -done in the `run-make-fulldeps` tests, with sample Rust programs (to be -instrumented) in the [`coverage`][coverage-test-samples] directory, and the -actual tests and expected results in [`coverage-reports`]. +[(See also the compiletest documentation for the `tests/coverage` +test suite.)](./tests/compiletest.md#coverage-tests) -Finally, the [`coverage-llvmir`] test compares compiles a simple Rust program +Coverage instrumentation in the MIR is validated by a `mir-opt` test: +[`tests/mir-opt/instrument_coverage.rs`]. + +Coverage instrumentation in LLVM IR is validated by the [`tests/coverage`] +test suite in `coverage-map` mode. +These tests compile a test program to LLVM IR assembly, and then +use the [`src/tools/coverage-dump`] tool to extract and pretty-print the +coverage mappings that would be embedded in the final binary. + +End-to-end testing of coverage instrumentation and coverage reporting is +performed by the [`tests/coverage`] test suite in `coverage-run` mode, +and by the [`tests/coverage-run-rustdoc`] test suite. +These tests compile and run a test program with coverage +instrumentation, then use LLVM tools to convert the coverage data into a +human-readable coverage report. + +Finally, the [`coverage-llvmir`] test compiles a simple Rust program with `-C instrument-coverage` and compares the compiled program's LLVM IR to expected LLVM IR instructions and structured data for a coverage-enabled program, including various checks for Coverage Map-related metadata and the LLVM intrinsic calls to increment the runtime counters. -Expected results for both the `mir-opt` tests and the `coverage*` tests under -`run-make-fulldeps` can be refreshed by running: +Expected results for the `coverage`, `coverage-run-rustdoc`, +and `mir-opt` tests can be refreshed by running: ```shell -$ ./x.py test mir-opt --bless -$ ./x.py test src/test/run-make-fulldeps/coverage --bless +./x test coverage --bless +./x test coverage-run-rustdoc --bless +./x test tests/mir-opt --bless ``` -[mir-opt-test]: https://github.com/rust-lang/rust/blob/master/src/test/mir-opt/instrument_coverage.rs -[coverage-test-samples]: https://github.com/rust-lang/rust/tree/master/src/test/run-make-fulldeps/coverage -[`coverage-reports`]: https://github.com/rust-lang/rust/tree/master/src/test/run-make-fulldeps/coverage-reports -[`coverage-spanview`]: https://github.com/rust-lang/rust/tree/master/src/test/run-make-fulldeps/coverage-spanview -[spanview-debugging]: compiler-debugging.md#viewing-spanview-output -[`coverage-llvmir`]: https://github.com/rust-lang/rust/tree/master/src/test/run-make-fulldeps/coverage-llvmir +[`tests/mir-opt/instrument_coverage.rs`]: https://github.com/rust-lang/rust/blob/master/tests/mir-opt/instrument_coverage.rs +[`tests/coverage`]: https://github.com/rust-lang/rust/tree/master/tests/coverage +[`src/tools/coverage-dump`]: https://github.com/rust-lang/rust/tree/master/src/tools/coverage-dump +[`tests/coverage-run-rustdoc`]: https://github.com/rust-lang/rust/tree/master/tests/coverage-run-rustdoc +[`coverage-llvmir`]: https://github.com/rust-lang/rust/tree/master/tests/run-make/coverage-llvmir ## Implementation Details of the `InstrumentCoverage` MIR Pass @@ -316,39 +322,33 @@ function, generic, or closure), the `Instrumentor`'s constructor prepares a [`inject_counters()`][inject-counters]. ```rust - Instrumentor::new(&self.name(), tcx, mir_body).inject_counters(); +Instrumentor::new(&self.name(), tcx, mir_body).inject_counters(); ``` The `CoverageGraph` is a coverage-specific simplification of the MIR control flow graph (CFG). Its nodes are [`BasicCoverageBlock`s][bcb], which encompass one or more sequentially-executed MIR `BasicBlock`s -(with no internal branching), plus a `CoverageKind` counter (to -be added, via coverage analysis), and an optional set of additional counters -to count incoming edges (if there are more than one). +(with no internal branching). + +Nodes and edges in the graph can have associated [`BcbCounter`]s, which are +stored in [`CoverageCounters`]. The `Instrumentor`'s `inject_counters()` uses the `CoverageGraph` to compute the best places to inject coverage counters, as MIR `Statement`s, with the following steps: -1. Depending on the debugging configurations in `rustc`'s, `config.toml`, - and `rustc` command line flags, various debugging features may be enabled - to enhance `debug!()` messages in logs, and to generate various "dump" files, - to help developers understand the MIR transformation process for coverage. - Most of the debugging features are implemented in the [`debug`][debug] - sub-module. -2. [`generate_coverage_spans()`][generate-coverage-spans] computes the minimum set of distinct, +1. [`generate_coverage_spans()`][generate-coverage-spans] computes the minimum set of distinct, non-branching code regions, from the MIR. These `CoverageSpan`s represent a span of code that must be counted. -3. [`make_bcb_counters()`][make-bcb-counters] generates `CoverageKind::Counter`s and - `CoverageKind::Expression`s for each `CoverageSpan`, plus additional - `intermediate_expressions`[^intermediate-expressions], not associated with any `CodeRegion`, but +2. [`make_bcb_counters()`][make-bcb-counters] generates `BcbCounter::Counter`s and + `BcbCounter::Expression`s for each `CoverageSpan`, plus additional + _intermediate expressions_[^intermediate-expressions] that are not associated + with any `CodeRegion`, but are required to compute a final `Expression` value for a `CodeRegion`. -4. Inject the new counters into the MIR, as new `StatementKind::Coverage` - statements. This is done by three distinct functions: - - `inject_coverage_span_counters()` - - `inject_indirect_counters()` - - `inject_intermediate_expression()`, called for each intermediate expression - returned from `make_bcb_counters()` +3. Inject the new counters into the MIR, as new `StatementKind::Coverage` + statements. +4. Attach all other necessary coverage information to the function's body as + [`FunctionCoverageInfo`]. [^intermediate-expressions]: Intermediate expressions are sometimes required because `Expression`s are limited to binary additions or subtractions. For @@ -360,7 +360,8 @@ intermediate expression for `B - C`. [coverage-graph]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/graph/struct.CoverageGraph.html [inject-counters]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/struct.Instrumentor.html#method.inject_counters [bcb]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/graph/struct.BasicCoverageBlock.html -[debug]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/debug +[`BcbCounter`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/counters/enum.BcbCounter.html +[`CoverageCounters`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/counters/struct.CoverageCounters.html [generate-coverage-spans]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/spans/struct.CoverageSpans.html#method.generate_coverage_spans [make-bcb-counters]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/counters/struct.BcbCounters.html#method.make_bcb_counters @@ -415,66 +416,12 @@ The BCB CFG is critical to simplifying the coverage analysis by ensuring graph p queries (`is_dominated_by()`, `predecessors`, `successors`, etc.) have branch (control flow) significance. -To visualize the `CoverageGraph`, you can generate a _graphviz_ `*.dot` -file with the following `rustc` flags:[^graphviz-dark-mode] - -[^graphviz-dark-mode]: This image also applies `-Z graphviz-dark-mode`, to -produce a Graphviz document with "dark mode" styling. If you use a dark mode or -theme in your development environment, you will probably want to use this -option so you can review the graphviz output without straining your vision. - -```shell -$ rustc -C instrument-coverage -Z dump-mir=InstrumentCoverage \ - -Z dump-mir-graphviz some_rust_source.rs -``` - -The `-Z dump-mir` flag requests [MIR debugging -output][mir-debugging] (generating `*.mir` files, by default). -`-Z dump-mir-graphviz` additionally generates `*.dot` files. -`-Z dump-mir=InstrumentCoverage` restricts these files to the -`InstrumentCoverage` pass. All files are written to the `./mir_dump/` -directory, by default. - -Files with names ending in `.-------.InstrumentCoverage.0.dot` contain the -_graphviz_ representations of a `CoverageGraph` (one for each MIR, that is, -for each function and closure): - -cropped image of a sample CoverageGraph in graphviz format -
- -This image shows each `BasicCoverageBlock` as a rectangular _node_, with -directional edges (the arrows) leading from each node to its `successors()`. -The nodes contain information in sections: - -1. The gray header has a label showing the BCB ID (or _index_ for looking up - its `BasicCoverageBlockData`). -2. The first content section shows the assigned `Counter` or `Expression` for - each contiguous section of code. (There may be more than one `Expression` - incremented by the same `Counter` for noncontiguous sections of code - representing the same sequential actions.) Note the code is represented by - the line and column ranges (for example: `52:28-52:33`, representing the - original source line 52, for columns 28-33). These are followed by the MIR - `Statement` or `Terminator` represented by that source range. (How these - coverage regions are determined is discussed in the following section.) -3. The final section(s) show the MIR `BasicBlock`s (by ID/index and its - `TerminatorKind`) contained in this BCB. The last BCB is separated out - because its `successors()` determine the edges leading out of the BCB, and - into the `leading_bb()` (first `BasicBlock`) of each successor BCB. - -Note, to find the `BasicCoverageBlock` from a final BCB `Terminator`'s -successor `BasicBlock`, there is an index and helper -function--[`bcb_from_bb()`][bcb-from-bb]--to look up a `BasicCoverageBlock` from -*any* contained `BasicBlock`. - [directed-graph]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/graph/trait.DirectedGraph.html [graph-traits]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/graph/index.html#traits [mir-dev-guide]: mir/index.md [compute-basic-coverage-blocks]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/graph/struct.CoverageGraph.html#method.compute_basic_coverage_blocks -[simplify-cfg]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/simplify/struct.SimplifyCfg.html +[simplify-cfg]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/simplify/enum.SimplifyCfg.html [rust-lang/rust#78544]: https://github.com/rust-lang/rust/issues/78544 -[mir-debugging]: mir/debugging.md -[bcb-from-bb]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/graph/struct.CoverageGraph.html#method.bcb_from_bb ### `CoverageSpans` @@ -493,32 +440,6 @@ The final stage of `generate_coverage_spans()` is handled by merges and de-duplicates them, and returns an optimal, minimal set of `CoverageSpan`s that can be used to assign coverage `Counter`s or `Expression`s, one-for-one. -An visual, interactive representation of the final `CoverageSpan`s can be -generated with the following `rustc` flags: - -```shell -$ rustc -C instrument-coverage -Z dump-mir=InstrumentCoverage \ - -Z dump-mir-spanview some_rust_source.rs -``` - -These flags request Spanview output for the `InstrumentCoverage` pass, and the -resulting files (one for each MIR, that is, for each function or closure) can be -found in the `mir_dump` directory (by default), with the extension: -`.-------.InstrumentCoverage.0.html`. - -cropped image of a sample Spanview in a browser -
- -The image above shows one such example. The orange and blue backgrounds -highlight alternating `CoverageSpan`s from the refined set. Hovering over a -line expands the output on that line to show the MIR `BasicBlock` IDs covered -by each `CoverageSpan`. While hovering, the `CoverageSpan` under the pointer -also has a _tooltip_ block of text, showing even more detail, including the -MIR `Statement`s and `Terminator`s contributing to the `CoverageSpan`, and -their individual `Span`s (which should be encapsulated within the code region -of the refined `CoverageSpan`) - [coverage-spans]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/spans/struct.CoverageSpans.html [coverage-span]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/spans/struct.CoverageSpan.html [to-refined-spans]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/spans/struct.CoverageSpans.html#method.to_refined_spans @@ -586,43 +507,3 @@ its `Counter` or `Expression`. [bcb-counters]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/counters/struct.BcbCounters.html [traverse-coverage-graph-with-loops]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/graph/struct.TraverseCoverageGraphWithLoops.html - -### Injecting counters into a MIR `BasicBlock` - -With the refined `CoverageSpan`s, and after all `Counter`s and `Expression`s are -created, the final step is to inject the `StatementKind::Coverage` statements -into the MIR. There are three distinct sources, handled by the following -functions: - -- [`inject_coverage_span_counters()`][inject-coverage-span-counters] injects the - counter from each `CoverageSpan`'s BCB. -- [`inject_indirect_counters()`][inject-indirect-counters] injects counters - for any BCB not assigned to a `CoverageSpan`, and for all edge counters. - These counters don't have `CoverageSpan`s. -- [`inject_intermediate_expression()`][inject-intermediate-expression] injects - the intermediate expressions returned from `make_bcb_counters()`. These - counters aren't associated with any BCB, edge, or `CoverageSpan`. - -These three functions inject the `Coverage` statements into the MIR. -`Counter`s and `Expression`s with `CoverageSpan`s add `Coverage` statements -to a corresponding `BasicBlock`, with a `CodeRegion` computed from the -refined `Span` and current `SourceMap`. - -All other `Coverage` statements have a `CodeRegion` of `None`, but they -still must be injected because they contribute to other `Expression`s. - -Finally, edge's with a `CoverageKind::Counter` require a new `BasicBlock`, -so the counter is only incremented when traversing the branch edge. - -[inject-coverage-span-counters]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/struct.Instrumentor.html#method.inject_coverage_span_counters -[inject-indirect-counters]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/struct.Instrumentor.html#method.inject_indirect_counters -[inject-intermediate-expression]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/fn.inject_intermediate_expression.html - -### Additional Debugging Support - -See the -[crate documentation for `rustc_mir::transform::coverage::debug`][coverage-debugging] -for a detailed description of the debug output, logging, and configuration options -available to developers working on the `InstrumentCoverage` pass. - -[coverage-debugging]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/debug/index.html diff --git a/src/macro-expansion.md b/src/macro-expansion.md index 7f50f7f89..f3883223d 100644 --- a/src/macro-expansion.md +++ b/src/macro-expansion.md @@ -48,7 +48,7 @@ iteration, this represents a compile error. Here is the [algorithm][original]: [fef]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_expand/expand/struct.MacroExpander.html#method.fully_expand_fragment [original]: https://github.com/rust-lang/rust/pull/53778#issuecomment-419224049 -1. Initialize an `queue` of unresolved macros. +1. Initialize a `queue` of unresolved macros. 2. Repeat until `queue` is empty (or we make no progress, which is an error): 1. [Resolve](./name-resolution.md) imports in our partially built crate as much as possible. diff --git a/src/memory.md b/src/memory.md index e5122deb2..4072d4f33 100644 --- a/src/memory.md +++ b/src/memory.md @@ -40,16 +40,16 @@ to that buffer is freed and our `'tcx` references would be invalid. In addition to types, there are a number of other arena-allocated data structures that you can allocate, and which are found in this module. Here are a few examples: -- [`Substs`][subst], allocated with `mk_substs` – this will intern a slice of types, often used to - specify the values to be substituted for generics (e.g. `HashMap` would be represented - as a slice `&'tcx [tcx.types.i32, tcx.types.u32]`). +- [`GenericArgs`], allocated with `mk_args` – this will intern a slice of types, often used +to specify the values to be substituted for generics args (e.g. `HashMap` would be +represented as a slice `&'tcx [tcx.types.i32, tcx.types.u32]`). - [`TraitRef`], typically passed by value – a **trait reference** consists of a reference to a trait along with its various type parameters (including `Self`), like `i32: Display` (here, the def-id - would reference the `Display` trait, and the substs would contain `i32`). Note that `def-id` is + would reference the `Display` trait, and the args would contain `i32`). Note that `def-id` is defined and discussed in depth in the `AdtDef and DefId` section. - [`Predicate`] defines something the trait system has to prove (see `traits` module). -[subst]: ./generic_arguments.html#subst +[`GenericArgs`]: ./generic_arguments.html#GenericArgs [`TraitRef`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.TraitRef.html [`Predicate`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Predicate.html diff --git a/src/mir/debugging.md b/src/mir/debugging.md index 3da17785b..4e1d68938 100644 --- a/src/mir/debugging.md +++ b/src/mir/debugging.md @@ -14,10 +14,10 @@ elements obscured by overlaps, and tooltips to view the MIR statements). This flag takes an optional value: `statement` (the default), `terminator`, or `block`, to generate span highlights with different levels of granularity. -`-Z dump-mir=F` is a handy compiler options that will let you view the MIR for +`-Z dump-mir=F` is a handy compiler option that will let you view the MIR for each function at each stage of compilation. `-Z dump-mir` takes a **filter** `F` which allows you to control which functions and which passes you are -interesting in. For example: +interested in. For example: ```bash > rustc -Z dump-mir=foo ... diff --git a/src/mir/index.md b/src/mir/index.md index 0c00928c0..36733c598 100644 --- a/src/mir/index.md +++ b/src/mir/index.md @@ -62,6 +62,12 @@ show you the MIR for your program. Try putting this program into play button on the top: [sample-play]: https://play.rust-lang.org/?gist=30074856e62e74e91f06abd19bd72ece&version=stable +MIR shown by above link is optimized. +Some statements like `StorageLive` are removed in optimization. +This happens because the compiler notices the value is never accessed in the code. +We can use `rustc [filename].rs -Z mir-opt-level=0 --emit mir` to view unoptimized MIR. +This requires the nightly toolchain. + ```rust fn main() { @@ -217,7 +223,7 @@ over the overflow checks.) ## MIR data types The MIR data types are defined in the [`compiler/rustc_middle/src/mir/`][mir] -module. Each of the key concepts mentioned in the previous section +module. Each of the key concepts mentioned in the previous section maps in a fairly straightforward way to a Rust type. The main MIR data type is [`Body`]. It contains the data for a single @@ -233,15 +239,14 @@ but [you can read about those below](#promoted)). - **Terminators** are represented by the [`Terminator`]. - **Locals** are represented by a [newtype'd] index type [`Local`]. The data for a local variable is found in the - [`Body::local_decls`][localdecls] vector). There is also a special constant + [`Body::local_decls`][localdecls] vector. There is also a special constant [`RETURN_PLACE`] identifying the special "local" representing the return value. -- **Places** are identified by the enum [`Place`]. There are a few - variants: +- **Places** are identified by the struct [`Place`]. There are a few + fields: - Local variables like `_1` - - Static variables `FOO` - **Projections**, which are fields or other things that "project - out" from a base place. These are represented by the type - [`ProjectionElem`]. So e.g. the place `_1.f` is a projection, + out" from a base place. These are represented by the [newtype'd] type + [`ProjectionElem`]. So e.g. the place `_1.f` is a projection, with `f` being the "projection element" and `_1` being the base path. `*_1` is also a projection, with the `*` being represented by the [`ProjectionElem::Deref`] element. @@ -250,7 +255,75 @@ but [you can read about those below](#promoted)). ## Representing constants -*to be written* +When code has reached the MIR stage, constants can generally come in two forms: +*MIR constants* ([`mir::Constant`]) and *type system constants* ([`ty::Const`]). +MIR constants are used as operands: in `x + CONST`, `CONST` is a MIR constant; +similarly, in `x + 2`, `2` is a MIR constant. Type system constants are used in +the type system, in particular for array lengths but also for const generics. + +Generally, both kinds of constants can be "unevaluated" or "already evaluated". +And unevaluated constant simply stores the `DefId` of what needs to be evaluated +to compute this result. An evaluated constant (a "value") has already been +computed; their representation differs between type system constants and MIR +constants: MIR constants evaluate to a `mir::ConstValue`; type system constants +evaluate to a `ty::ValTree`. + +Type system constants have some more variants to support const generics: they +can refer to local const generic parameters, and they are subject to inference. +Furthermore, the `mir::Constant::Ty` variant lets us use an arbitrary type +system constant as a MIR constant; this happens whenever a const generic +parameter is used as an operand. + +### MIR constant values + +In general, a MIR constant value (`mir::ConstValue`) was computed by evaluating +some constant the user wrote. This [const evaluation](../const-eval.md) produces +a very low-level representation of the result in terms of individual bytes. We +call this an "indirect" constant (`mir::ConstValue::Indirect`) since the value +is stored in-memory. + +However, storing everything in-memory would be awfully inefficient. Hence there +are some other variants in `mir::ConstValue` that can represent certain simple +and common values more efficiently. In particular, everything that can be +directly written as a literal in Rust (integers, floats, chars, bools, but also +`"string literals"` and `b"byte string literals"`) has an optimized variant that +avoids the full overhead of the in-memory representation. + +### ValTrees + +An evaluated type system constant is a "valtree". The `ty::ValTree` datastructure +allows us to represent + +* arrays, +* many structs, +* tuples, +* enums and, +* most primitives. + +The most important rule for +this representation is that every value must be uniquely represented. In other +words: a specific value must only be representable in one specific way. For example: there is only +one way to represent an array of two integers as a `ValTree`: +`ValTree::Branch(&[ValTree::Leaf(first_int), ValTree::Leaf(second_int)])`. +Even though theoretically a `[u32; 2]` could be encoded in a `u64` and thus just be a +`ValTree::Leaf(bits_of_two_u32)`, that is not a legal construction of `ValTree` +(and is very complex to do, so it is unlikely anyone is tempted to do so). + +These rules also mean that some values are not representable. There can be no `union`s in type +level constants, as it is not clear how they should be represented, because their active variant +is unknown. Similarly there is no way to represent raw pointers, as addresses are unknown at +compile-time and thus we cannot make any assumptions about them. References on the other hand +*can* be represented, as equality for references is defined as equality on their value, so we +ignore their address and just look at the backing value. We must make sure that the pointer values +of the references are not observable at compile time. We thus encode `&42` exactly like `42`. +Any conversion from +valtree back a to MIR constant value must reintroduce an actual indirection. At codegen time the +addresses may be deduplicated between multiple uses or not, entirely depending on arbitrary +optimization choices. + +As a consequence, all decoding of `ValTree` must happen by matching on the type first and making +decisions depending on that. The value itself gives no useful information without the type that +belongs to it. @@ -278,3 +351,5 @@ See the const-eval WG's [docs on promotion](https://github.com/rust-lang/const-e [`ProjectionElem::Deref`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.ProjectionElem.html#variant.Deref [`Rvalue`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.Rvalue.html [`Operand`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.Operand.html +[`mir::Constant`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/struct.Constant.html +[`ty::Const`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Const.html diff --git a/src/mir/optimizations.md b/src/mir/optimizations.md index cef785ea9..16cf2035b 100644 --- a/src/mir/optimizations.md +++ b/src/mir/optimizations.md @@ -28,13 +28,13 @@ optimizes it, and returns the improved MIR. ## Quickstart for adding a new optimization -1. Make a Rust source file in `src/test/mir-opt` that shows the code you want to +1. Make a Rust source file in `tests/mir-opt` that shows the code you want to optimize. This should be kept simple, so avoid `println!` or other formatting code if it's not necessary for the optimization. The reason for this is that `println!`, `format!`, etc. generate a lot of MIR that can make it harder to understand what the optimization does to the test. -2. Run `./x.py test --bless src/test/mir-opt/.rs` to generate a MIR +2. Run `./x test --bless tests/mir-opt/.rs` to generate a MIR dump. Read [this README][mir-opt-test-readme] for instructions on how to dump things. @@ -45,16 +45,16 @@ optimizes it, and returns the improved MIR. 4. Implement a new optimization in [`compiler/rustc_mir_transform/src`]. The fastest and easiest way to do this is to - 1. pick a small optimization (such as [`no_landing_pads`]) and copy it + 1. pick a small optimization (such as [`remove_storage_markers`]) and copy it to a new file, 2. add your optimization to one of the lists in the [`run_optimization_passes()`] function, 3. and then start modifying the copied optimization. -5. Rerun `./x.py test --bless src/test/mir-opt/.rs` to regenerate the +5. Rerun `./x test --bless tests/mir-opt/.rs` to regenerate the MIR dumps. Look at the diffs to see if they are what you expect. -6. Run `./x.py test src/test/ui` to see if your optimization broke anything. +6. Run `./x test tests/ui` to see if your optimization broke anything. 7. If there are issues with your optimization, experiment with it a bit and repeat steps 5 and 6. @@ -70,10 +70,9 @@ optimizes it, and returns the improved MIR. If you have any questions along the way, feel free to ask in `#t-compiler/wg-mir-opt` on Zulip. -[mir-opt-test-readme]: https://github.com/rust-lang/rust/blob/master/src/test/mir-opt/README.md +[mir-opt-test-readme]: https://github.com/rust-lang/rust/blob/master/tests/mir-opt/README.md [`compiler/rustc_mir_transform/src`]: https://github.com/rust-lang/rust/tree/master/compiler/rustc_mir_transform/src - -[`no_landing_pads`]: https://github.com/rust-lang/rust/blob/master/compiler/rustc_mir_transform/src/no_landing_pads.rs +[`remove_storage_markers`]: https://github.com/rust-lang/rust/blob/HEAD/compiler/rustc_mir_transform/src/remove_storage_markers.rs [`run_optimization_passes()`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/fn.run_optimization_passes.html ## Defining optimization passes @@ -131,6 +130,6 @@ needed for correctness), then fuel should not be used. An example of this is `Pr perform optimizations. In these situations, we should allow for the initial gathering pass and then check fuel as close to the mutation as possible. This allows for the best debugging experience, because we can determine where in the list of candidates an optimization -may have been misapplied. Examples of this are `InstCombine` and `ConstantPropagation`. +may have been misapplied. Examples of this are `InstSimplify` and `ConstantPropagation`. [consideroptimizing]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.consider_optimizing diff --git a/src/mir/visitor.md b/src/mir/visitor.md index 505b700b3..c0c703a6b 100644 --- a/src/mir/visitor.md +++ b/src/mir/visitor.md @@ -23,7 +23,7 @@ struct MyVisitor<...> { and you then implement the `Visitor` or `MutVisitor` trait for that type: ```rust,ignore -impl<'tcx> MutVisitor<'tcx> for NoLandingPads { +impl<'tcx> MutVisitor<'tcx> for MyVisitor { fn visit_foo(&mut self, ...) { ... self.super_foo(...); @@ -37,10 +37,10 @@ code that will execute whenever a `foo` is found. If you want to recursively walk the contents of the `foo`, you then invoke the `super_foo` method. (NB. You never want to override `super_foo`.) -A very simple example of a visitor can be found in [`LocalUseCounter`]. -By implementing `visit_local` method, this visitor counts how many times each local is used. +A very simple example of a visitor can be found in [`LocalUseVisitor`]. +By implementing `visit_local` method, this visitor counts how many times each local is mutably used. -[`LocalUseCounter`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/simplify_try/struct.LocalUseCounter.html +[`LocalUseVisitor`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/const_debuginfo/struct.LocalUseVisitor.html ## Traversal diff --git a/src/name-resolution.md b/src/name-resolution.md index 1dbc95ead..fbeef19ff 100644 --- a/src/name-resolution.md +++ b/src/name-resolution.md @@ -91,8 +91,8 @@ part of another, it doesn't mean the name visible in the outer one is also visible in the inner one, or that it refers to the same thing. To cope with that, the compiler introduces the concept of Ribs. This is -abstraction of a scope. Every time the set of visible names potentially changes, -a new rib is pushed onto a stack. The places where this can happen includes for +an abstraction of a scope. Every time the set of visible names potentially changes, +a new rib is pushed onto a stack. The places where this can happen include for example: * The obvious places ‒ curly braces enclosing a block, function boundaries, @@ -103,8 +103,8 @@ example: When searching for a name, the stack of ribs is traversed from the innermost outwards. This helps to find the closest meaning of the name (the one not -shadowed by anything else). The transition to outer rib may also change the -rules what names are usable ‒ if there are nested functions (not closures), +shadowed by anything else). The transition to outer rib may also affect +what names are usable ‒ if there are nested functions (not closures), the inner one can't access parameters and local bindings of the outer one, even though they should be visible by ordinary scoping rules. An example: @@ -117,7 +117,7 @@ fn do_something(val: T) { // <- New rib in both types and values (1) }; // End of (3) // `val` is accessible, `helper` variable shadows `helper` function fn helper() { // <- New rib in both types and values (4) - // `val` is not accessible here, (4) is not transparent for locals) + // `val` is not accessible here, (4) is not transparent for locals // `T` is not accessible here } // End of (4) let val = T::default(); // New rib (5) diff --git a/src/notification-groups/about.md b/src/notification-groups/about.md index 260c59c24..1307a50fc 100644 --- a/src/notification-groups/about.md +++ b/src/notification-groups/about.md @@ -14,7 +14,7 @@ Of course, you don't have to wait for new issues to be tagged! If you prefer, you can use the Github label for a notification group to search for existing issues that haven't been claimed yet. -[claim the issue]: https://github.com/rust-lang/triagebot/wiki/Assignment +[claim the issue]: https://forge.rust-lang.org/triagebot/issue-assignment.html ## List of notification groups @@ -42,7 +42,7 @@ particularly those of **middle priority**: ## Joining a notification group -To join an notification group, you just have to open a PR adding your +To join a notification group, you just have to open a PR adding your Github username to the appropriate file in the Rust team repository. See the "example PRs" below to get a precise idea and to identify the file to edit. @@ -95,5 +95,5 @@ or contributors, and is typically done as part of compiler team triage.** [rustbot]: https://github.com/rust-lang/triagebot/ -[`ping`]: https://github.com/rust-lang/triagebot/wiki/Pinging +[`ping`]: https://forge.rust-lang.org/triagebot/pinging.html [`triagebot.toml`]: https://github.com/rust-lang/rust/blob/master/triagebot.toml diff --git a/src/notification-groups/cleanup-crew.md b/src/notification-groups/cleanup-crew.md index 436b51fd1..707334304 100644 --- a/src/notification-groups/cleanup-crew.md +++ b/src/notification-groups/cleanup-crew.md @@ -39,9 +39,9 @@ are a few: * The [rust-reduce](https://github.com/jethrogb/rust-reduce) tool can try to reduce code automatically. - * The [C-reduce](https://embed.cs.utah.edu/creduce/) tool also works + * The [C-reduce](https://github.com/csmith-project/creduce) tool also works on Rust code, though it requires that you start from a single - file. (XXX link to some post explaining how to do it?) + file. (A post explaining how to do it can be found [here](https://insaneinside.net/2017/09/12/whole-crate-bug-reduction-with-creduce.html).) * pnkfelix's [Rust Bug Minimization Patterns] blog post * This post focuses on "heavy bore" techniques, where you are starting with a large, complex cargo project that you wish to @@ -77,38 +77,13 @@ various builds of rustc. For recent regressions, it is even able to use the builds from our CI to track down the regression to a specific PR; for older regressions, it will simply identify a nightly. -To learn to use [cargo-bisect-rustc], check out [this blog -post][learn], which gives a quick introduction to how it works. You -can also ask questions at the Zulip stream -[`#t-compiler/cargo-bisect-rustc`][zcbr], or help in improving the tool. +To learn to use [cargo-bisect-rustc], check out [this blog post][learn], which +gives a quick introduction to how it works. Additionally, there is a [Guide] +which goes into more detail on how to use it. You can also ask questions at +the Zulip stream [`#t-compiler/cargo-bisect-rustc`][zcbr], or help in +improving the tool. [cargo-bisect-rustc]: https://github.com/rust-lang/cargo-bisect-rustc/ [learn]: https://blog.rust-lang.org/inside-rust/2019/12/18/bisecting-rust-compiler.html [zcbr]: https://rust-lang.zulipchat.com/#narrow/stream/217417-t-compiler.2Fcargo-bisect-rustc - -### identifying the range of PRs in a nightly - -If the regression occurred more than 90 days ago, then -cargo-bisect-rustc will not able to identify the particular PR that -caused the regression, just the nightly build. In that case, we can -identify the set of PRs that this corresponds to by using the git -history. - -The command `rustc +nightly -vV` will cause rustc to output a number -of useful bits of version info, including the `commit-hash`. Given the -commit-hash of two nightly versions, you can find all of PRs that have -landed in between by taking the following steps: - -1. Go to an update checkout of the [rust-lang/rust] repository -2. Execute the command `git log --author=bors --format=oneline SHA1..SHA2` - * This will list out all of the commits by bors, which is our merge bot - * Each commit corresponds to one PR, and information about the PR should be in the description -3. Copy and paste that information into the bug report - -Often, just eye-balling the PR descriptions (which are included in the -commit messages) will give you a good idea which one likely caused the -problem. But if you're unsure feel free to just ping the compiler team -(`@rust-lang/compiler`) or else to ping the authors of the PR -themselves. - -[rust-lang/rust]: https://github.com/rust-lang/rust/ +[Guide]: https://rust-lang.github.io/cargo-bisect-rustc/ diff --git a/src/overview.md b/src/overview.md index ca37f94b2..797f4d8e2 100644 --- a/src/overview.md +++ b/src/overview.md @@ -1,4 +1,4 @@ -# Overview of the Compiler +# Overview of the compiler @@ -286,7 +286,7 @@ of a crate.][passes] Then the codegen backend invokes the for that function and then creates codegen units. This kind of split will need to remain to ensure that unreachable functions still have their errors emitted. -[passes]: https://github.com/rust-lang/rust/blob/45ebd5808afd3df7ba842797c0fcd4447ddf30fb/src/librustc_interface/passes.rs#L824 +[passes]: https://github.com/rust-lang/rust/blob/e69c7306e2be08939d95f14229e3f96566fb206c/compiler/rustc_interface/src/passes.rs#L791 Moreover, the compiler wasn't originally built to use a query system; the query system has been retrofitted into the compiler, so parts of it are not query-fied diff --git a/src/parallel-rustc.md b/src/parallel-rustc.md index e7cbbd7ee..9942f751a 100644 --- a/src/parallel-rustc.md +++ b/src/parallel-rustc.md @@ -44,7 +44,6 @@ are implemented differently depending on whether `parallel-compiler` is true. | MappedWriteGuard | parking_lot::MappedRwLockWriteGuard | std::cell::RefMut | | LockGuard | parking_lot::MutexGuard | std::cell::RefMut | | MappedLockGuard | parking_lot::MappedMutexGuard | std::cell::RefMut | -| MetadataRef | [`OwningRef, [u8]>`][OwningRef] | [`OwningRef, [u8]>`][OwningRef] | - These thread-safe data structures interspersed during compilation can cause a lot of lock contention, which actually degrades performance as the @@ -164,4 +163,3 @@ are a bit out of date): [parallel-rustdoc]: https://github.com/rust-lang/rust/issues/82741 [Arc]: https://doc.rust-lang.org/std/sync/struct.Arc.html [Rc]: https://doc.rust-lang.org/std/rc/struct.Rc.html -[OwningRef]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_data_structures/owning_ref/index.html diff --git a/src/part-2-intro.md b/src/part-2-intro.md index 5ea4d7b6b..aecab562b 100644 --- a/src/part-2-intro.md +++ b/src/part-2-intro.md @@ -1,4 +1,4 @@ -# Part 2: High-Level Compiler Architecture +# High-Level Compiler Architecture The remaining parts of this guide discuss how the compiler works. They go through everything from high-level structure of the compiler to how each stage diff --git a/src/part-3-intro.md b/src/part-3-intro.md index c10080632..59a1eeba7 100644 --- a/src/part-3-intro.md +++ b/src/part-3-intro.md @@ -1,4 +1,4 @@ -# Part 3: Source Code Representation +# Source Code Representation This part describes the process of taking raw source code from the user and transforming it into various forms that the compiler can work with easily. diff --git a/src/part-4-intro.md b/src/part-4-intro.md index 00a74f308..6a8433164 100644 --- a/src/part-4-intro.md +++ b/src/part-4-intro.md @@ -1,4 +1,4 @@ -# Part 4: Analysis +# Analysis This part discusses the many analyses that the compiler uses to check various properties of the code and to inform later stages. Typically, this is what people diff --git a/src/profile-guided-optimization.md b/src/profile-guided-optimization.md index db2624e92..69c0599a0 100644 --- a/src/profile-guided-optimization.md +++ b/src/profile-guided-optimization.md @@ -131,8 +131,8 @@ in [run-make tests][rmake-tests] (the relevant tests have `pgo` in their name). There is also a [codegen test][codegen-test] that checks that some expected instrumentation artifacts show up in LLVM IR. -[rmake-tests]: https://github.com/rust-lang/rust/tree/master/src/test/run-make-fulldeps -[codegen-test]: https://github.com/rust-lang/rust/blob/master/src/test/codegen/pgo-instrumentation.rs +[rmake-tests]: https://github.com/rust-lang/rust/tree/master/tests/run-make-fulldeps +[codegen-test]: https://github.com/rust-lang/rust/blob/master/tests/codegen/pgo-instrumentation.rs ## Additional Information diff --git a/src/profiling.md b/src/profiling.md index e1666e237..df987e00a 100644 --- a/src/profiling.md +++ b/src/profiling.md @@ -19,7 +19,7 @@ Depending on what you're trying to measure, there are several different approach - If you want a nice visual representation of the compile times of your crate graph, you can use [cargo's `--timings` flag](https://doc.rust-lang.org/nightly/cargo/reference/timings.html), e.g. `cargo build --timings`. - You can use this flag on the compiler itself with `CARGOFLAGS="--timings" ./x.py build` + You can use this flag on the compiler itself with `CARGOFLAGS="--timings" ./x build` - If you want to profile memory usage, you can use various tools depending on what operating system you are using. @@ -41,11 +41,11 @@ extension in LLVM bitcode format. Example usage: ``` cargo install cargo-llvm-lines -# On a normal crate you could now run `cargo llvm-lines`, but `x.py` isn't normal :P +# On a normal crate you could now run `cargo llvm-lines`, but `x` isn't normal :P # Do a clean before every run, to not mix in the results from previous runs. -./x.py clean -env RUSTFLAGS=-Csave-temps ./x.py build --stage 0 compiler/rustc +./x clean +env RUSTFLAGS=-Csave-temps ./x build --stage 0 compiler/rustc # Single crate, e.g., rustc_middle. (Relies on the glob support of your shell.) # Convert unoptimized LLVM bitcode into a human readable LLVM assembly accepted by cargo-llvm-lines. @@ -85,7 +85,7 @@ Example output for the compiler: 326903 (0.7%) 642 (0.0%) rustc_query_system::query::plumbing::try_execute_query ``` -Since this doesn't seem to work with incremental compilation or `./x.py check`, +Since this doesn't seem to work with incremental compilation or `./x check`, you will be compiling rustc _a lot_. I recommend changing a few settings in `config.toml` to make it bearable: ``` diff --git a/src/profiling/with_perf.md b/src/profiling/with_perf.md index af30b4732..469e237b9 100644 --- a/src/profiling/with_perf.md +++ b/src/profiling/with_perf.md @@ -10,7 +10,7 @@ This is a guide for how to profile rustc with [perf](https://perf.wiki.kernel.or - `debuginfo-level = 1` - enables line debuginfo - `jemalloc = false` - lets you do memory use profiling with valgrind - leave everything else the defaults -- Run `./x.py build` to get a full build +- Run `./x build` to get a full build - Make a rustup toolchain pointing to that result - see [the "build and run" section for instructions][b-a-r] @@ -90,14 +90,15 @@ You can also use that same command to use cachegrind or other profiling tools. If you prefer to run things manually, that is also possible. You first need to find the source for the test you want. Sources for the tests -are found in [the `collector/benchmarks` directory][dir]. So let's go -into the directory of a specific test; we'll use `clap-rs` as an -example: +are found in [the `collector/compile-benchmarks` directory][compile-time dir] +and [the `collector/runtime-benchmarks` directory][runtime dir]. So let's +go into the directory of a specific test; we'll use `clap-rs` as an example: -[dir]: https://github.com/rust-lang/rustc-perf/tree/master/collector/benchmarks +[compile-time dir]: https://github.com/rust-lang/rustc-perf/tree/master/collector/compile-benchmarks +[runtime dir]: https://github.com/rust-lang/rustc-perf/tree/master/collector/runtime-benchmarks ```bash -cd collector/benchmarks/clap-rs +cd collector/compile-benchmarks/clap-3.1.6 ``` In this case, let's say we want to profile the `cargo check` @@ -281,7 +282,7 @@ Tree What happens with `--tree-callees` is that - we find each sample matching the regular expression -- we look at the code that is occurs *after* the regex match and try +- we look at the code that occurs *after* the regex match and try to build up a call tree The `--tree-min-percent 3` option says "only show me things that take diff --git a/src/profiling/wpa_profiling.md b/src/profiling/wpa_profiling.md index e7cf9418c..a800c5717 100644 --- a/src/profiling/wpa_profiling.md +++ b/src/profiling/wpa_profiling.md @@ -47,11 +47,11 @@ we'll need to build a stage 1 compiler and then a stage 2 compiler ourselves. To do this, make sure you have set `debuginfo-level = 1` in your `config.toml` file. This tells rustc to generate debug information which includes stack frames when bootstrapping. -Now you can build the stage 1 compiler: `python x.py build --stage 1 -i library` or however +Now you can build the stage 1 compiler: `x build --stage 1 -i library` or however else you want to build the stage 1 compiler. Now that the stage 1 compiler is built, we can record the stage 2 build. Go back to WPR, click the -"start" button and build the stage 2 compiler (e.g., `python x build --stage=2 -i library`). +"start" button and build the stage 2 compiler (e.g., `x build --stage=2 -i library`). When this process finishes, stop the recording. Click the Save button and once that process is complete, click the "Open in WPA" button which diff --git a/src/queries/incremental-compilation-in-detail.md b/src/queries/incremental-compilation-in-detail.md index abd2b0155..76ebf10e8 100644 --- a/src/queries/incremental-compilation-in-detail.md +++ b/src/queries/incremental-compilation-in-detail.md @@ -62,7 +62,8 @@ the result of `type_of(bar)` might yield a different result than what we have in the cache and, transitively, the result of `type_check_item(foo)` might have changed too. We thus re-run `type_check_item(foo)`, which in turn will re-run `type_of(bar)`, which will yield an up-to-date result -because it reads the up-to-date version of `Hir(bar)`. +because it reads the up-to-date version of `Hir(bar)`. Also, we re-run +`type_check_item(bar)` because result of `type_of(bar)` might have changed. ## The Problem With The Basic Algorithm: False Positives diff --git a/src/queries/query-evaluation-model-in-detail.md b/src/queries/query-evaluation-model-in-detail.md index 8a08f1e04..296d88f6e 100644 --- a/src/queries/query-evaluation-model-in-detail.md +++ b/src/queries/query-evaluation-model-in-detail.md @@ -201,8 +201,8 @@ OK as long as the mutation is not observable. This is achieved by two things: - Before a result is stolen, we make sure to eagerly run all queries that might ever need to read that result. This has to be done manually by calling those queries. -- Whenever a query tries to access a stolen result, we make the compiler ICE so - that such a condition cannot go unnoticed. +- Whenever a query tries to access a stolen result, we make an ICE + (Internal Compiler Error) so that such a condition cannot go unnoticed. This is not an ideal setup because of the manual intervention needed, so it should be used sparingly and only when it is well known which queries might diff --git a/src/query.md b/src/query.md index 5dbffac8d..782c5b4b3 100644 --- a/src/query.md +++ b/src/query.md @@ -103,7 +103,7 @@ When the tcx is created, it is given the providers by its creator using the [`Providers`][providers_struct] struct. This struct is generated by the macros here, but it is basically a big list of function pointers: -[providers_struct]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/query/struct.Providers.html +[providers_struct]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/query/struct.Providers.html ```rust,ignore struct Providers { diff --git a/src/return-position-impl-trait-in-trait.md b/src/return-position-impl-trait-in-trait.md new file mode 100644 index 000000000..9ed62c87c --- /dev/null +++ b/src/return-position-impl-trait-in-trait.md @@ -0,0 +1,437 @@ +# Return Position Impl Trait In Trait + +Return-position impl trait in trait (RPITIT) is conceptually (and as of +[#112988], literally) sugar that turns RPITs in trait methods into +generic associated types (GATs) without the user having to define that +GAT either on the trait side or impl side. + +RPITIT was originally implemented in [#101224], which added support for +async fn in trait (AFIT), since the implementation for RPITIT came for +free as a part of implementing AFIT which had been RFC'd previously. It +was then RFC'd independently in [RFC 3425], which was recently approved +by T-lang. + +## How does it work? + +This doc is ordered mostly via the compilation pipeline. AST -> HIR -> +astconv -> typeck. + +### AST and HIR + +AST -> HIR lowering for RPITITs is almost the same as lowering RPITs. We +still lower them as +[`hir::ItemKind::OpaqueTy`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.OpaqueTy.html). +The two differences are that: + +We record `in_trait` for the opaque. This will signify that the opaque +is an RPITIT for astconv, diagnostics that deal with HIR, etc. + +We record `lifetime_mapping`s for the opaque type, described below. + +#### Aside: Opaque lifetime duplication + +*All opaques* (not just RPITITs) end up duplicating their captured +lifetimes into new lifetime parameters local to the opaque. The main +reason we do this is because RPITs need to be able to "reify"[^1] any +captured late-bound arguments, or make them into early-bound ones. This +is so they can be used as generic args for the opaque, and later to +instantiate hidden types. Since we don't know which lifetimes are early- +or late-bound during AST lowering, we just do this for all lifetimes. + +[^1]: This is compiler-errors terminology, I'm not claiming it's accurate :^) + +The main addition for RPITITs is that during lowering we track the +relationship between the captured lifetimes and the corresponding +duplicated lifetimes in an additional field, +[`OpaqueTy::lifetime_mapping`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir/hir/struct.OpaqueTy.html#structfield.lifetime_mapping). +We use this lifetime mapping later on in `predicates_of` to install +bounds that enforce equality between these duplicated lifetimes and +their source lifetimes in order to properly typecheck these GATs, which +will be discussed below. + +##### note: + +It may be better if we were able to lower without duplicates and for +that I think we would need to stop distinguishing between early and late +bound lifetimes. So we would need a solution like [Account for +late-bound lifetimes in generics +#103448](https://github.com/rust-lang/rust/pull/103448) and then also a +PR similar to [Inherit function lifetimes for impl-trait +#103449](https://github.com/rust-lang/rust/pull/103449). + +### Astconv + +The main change to astconv is that we lower `hir::TyKind::OpaqueDef` for +an RPITIT to a projection instead of an opaque, using a newly +synthesized def-id for a new associated type in the trait. We'll +describe how exactly we get this def-id in the next section. + +This means that any time we call `ast_ty_to_ty` on the RPITIT, we end up +getting a projection back instead of an opaque. This projection can then +be normalized to the right value -- either the original opaque if we're +in the trait, or the inferred type of the RPITIT if we're in an impl. + +#### Lowering to synthetic associated types + +Using query feeding, we synthesize new associated types on both the +trait side and impl side for RPITITs that show up in methods. + +##### Lowering RPITITs in traits + +When `tcx.associated_item_def_ids(trait_def_id)` is called on a trait to +gather all of the trait's associated types, the query previously just +returned the def-ids of the HIR items that are children of the trait. +After [#112988], additionally, for each method in the trait, we add the +def-ids returned by +`tcx.associated_types_for_impl_traits_in_associated_fn(trait_method_def_id)`, +which walks through each trait method, gathers any RPITITs that show up +in the signature, and then calls +`associated_type_for_impl_trait_in_trait` for each RPITIT, which +synthesizes a new associated type. + +##### Lowering RPITITs in impls + +Similarly, along with the impl's HIR items, for each impl method, we +additionally add all of the +`associated_types_for_impl_traits_in_associated_fn` for the impl method. +This calls `associated_type_for_impl_trait_in_impl`, which will +synthesize an associated type definition for each RPITIT that comes from +the corresponding trait method. + +#### Synthesizing new associated types + +We use query feeding +([`TyCtxtAt::create_def`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/query/plumbing/struct.TyCtxtAt.html#method.create_def)) +to synthesize a new def-id for the synthetic GATs for each RPITIT. + +Locally, most of rustc's queries match on the HIR of an item to compute +their values. Since the RPITIT doesn't really have HIR associated with +it, or at least not HIR that corresponds to an associated type, we must +compute many queries eagerly and +[feed](https://github.com/rust-lang/rust/pull/104940) them, like +`opt_def_kind`, `associated_item`, `visibility`, and`defaultness`. + +The values for most of these queries is obvious, since the RPITIT +conceptually inherits most of its information from the parent function +(e.g. `visibility`), or because it's trivially knowable because it's an +associated type (`opt_def_kind`). + +Some other queries are more involved, or cannot be feeded, and we +document the interesting ones of those below: + +##### `generics_of` for the trait + +The GAT for an RPITIT conceptually inherits the same generics as the +RPIT it comes from. However, instead of having the method as the +generics' parent, the trait is the parent. + +Currently we get away with taking the RPIT's generics and method +generics and flattening them both into a new generics list, preserving +the def-id of each of the parameters. (This may cause issues with +def-ids having the wrong parents, but in the worst case this will cause +diagnostics issues. If this ends up being an issue, we can synthesize +new def-ids for generic params whose parent is the GAT.) + +
+ An illustrated example + +```rust +trait Foo { + fn method<'early: 'early, 'late, T>() -> impl Sized + Captures<'early, 'late>; +} +``` + +Would desugar to... +```rust +trait Foo { + // vvvvvvvvv method's generics + // vvvvvvvvvvvvvvvvvvvvvvvv opaque's generics + type Gat<'early, T, 'early_duplicated, 'late>: Sized + Captures<'early_duplicated, 'late>; + + fn method<'early: 'early, 'late, T>() -> Self::Gat<'early, T, 'early, 'late>; +} +``` +
+ +##### `generics_of` for the impl + +The generics for an impl's GAT are a bit more interesting. They are +composed of RPITIT's own generics (from the trait definition), appended +onto the impl's methods generics. This has the same issue as above, +where the generics for the GAT have parameters whose def-ids have the +wrong parent, but this should only cause issues in diagnostics. + +We could fix this similarly if we were to synthesize new generics +def-ids, but this can be done later in a forwards-compatible way, +perhaps by a interested new contributor. + +##### `opt_rpitit_info` + +Some queries rely on computing information that would result in cycles +if we were to feed them eagerly, like `explicit_predicates_of`. +Therefore we defer to the `predicates_of` provider to return the right +value for our RPITIT's GAT. We do this by detecting early on in the +query if the associated type is synthetic by using +[`opt_rpitit_info`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.TyCtxt.html#method.opt_rpitit_info), +which returns `Some` if the associated type is synthetic. + +Then, during a query like `explicit_predicates_of`, we can detect if an +associated type is synthetic like: + +```rust +fn explicit_predicates_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ... { + if let Some(rpitit_info) = tcx.opt_rpitit_info(def_id) { + // Do something special for RPITITs... + return ...; + } + + // The regular computation which relies on access to the HIR of `def_id`. +} +``` + +##### `explicit_predicates_of` + +RPITITs begin by copying the predicates of the method that defined it, +both on the trait and impl side. + +Additionally, we install "bidirectional outlives" predicates. +Specifically, we add region-outlives predicates in both directions for +each captured early-bound lifetime that constrains it to be equal to the +duplicated early-bound lifetime that results from lowering. This is best +illustrated in an example: + +```rust +trait Foo<'a> { + fn bar() -> impl Sized + 'a; +} + +// Desugars into... + +trait Foo<'a> { + type Gat<'a_duplicated>: Sized + 'a + where + 'a: 'a_duplicated, + 'a_duplicated: 'a; + //~^ Specifically, we should be able to assume that the + // duplicated `'a_duplicated` lifetime always stays in + // sync with the `'a` lifetime. + + fn bar() -> Self::Gat<'a>; +} +``` + +##### `assumed_wf_types` + +The GATs in both the trait and impl inherit the `assumed_wf_types` of +the trait method that defines the RPITIT. This is to make sure that the +following code is well formed when lowered. + +```rust +trait Foo { + fn iter<'a, T>(x: &'a [T]) -> impl Iterator; +} + +// which is lowered to... + +trait FooDesugared { + type Iter<'a, T>: Iterator; + //~^ assumed wf: `&'a [T]` + // Without assumed wf types, the GAT would not be well-formed on its own. + + fn iter<'a, T>(x: &'a [T]) -> Self::Iter<'a, T>; +} +``` + +Because `assumed_wf_types` is only defined for local def ids, in order +to properly implement `assumed_wf_types` for impls of foreign traits +with RPITs, we need to encode the assumed wf types of RPITITs in an +extern query +[`assumed_wf_types_for_rpitit`](https://github.com/rust-lang/rust/blob/a17c7968b727d8413801961fc4e89869b6ab00d3/compiler/rustc_ty_utils/src/implied_bounds.rs#L14). + +### Typechecking + +#### The RPITIT inference algorithm + +The RPITIT inference algorithm is implemented in +[`collect_return_position_impl_trait_in_trait_tys`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/check/compare_impl_item/fn.collect_return_position_impl_trait_in_trait_tys.html). + +**High-level:** Given a impl method and a trait method, we take the +trait method and instantiate each RPITIT in the signature with an infer +var. We then equate this trait method signature with the impl method +signature, and process all obligations that fall out in order to infer +the type of all of the RPITITs in the method. + +The method is also responsible for making sure that the hidden types for +each RPITIT actually satisfy the bounds of the `impl Trait`, i.e. that +if we infer `impl Trait = Foo`, that `Foo: Trait` holds. + +
+ An example... + +```rust +#![feature(return_position_impl_trait_in_trait)] + +use std::ops::Deref; + +trait Foo { + fn bar() -> impl Deref; + // ^- RPITIT ?0 ^- RPITIT ?1 +} + +impl Foo for () { + fn bar() -> Box { Box::new(String::new()) } +} +``` + +We end up with the trait signature that looks like `fn() -> ?0`, and +nested obligations `?0: Deref`, `?1: Sized`. The impl +signature is `fn() -> Box`. + +Equating these signatures gives us `?0 = Box`, which then after +processing the obligation `Box: Deref` gives us `?1 += String`, and the other obligation `String: Sized` evaluates to true. + +By the end of the algorithm, we end up with a mapping between associated +type def-ids to concrete types inferred from the signature. We can then +use this mapping to implement `type_of` for the synthetic associated +types in the impl, since this mapping describes the type that should +come after the `=` in `type Assoc = ...` for each RPITIT. +
+ +##### Implied bounds in RPITIT hidden type inference + +Since `collect_return_position_impl_trait_in_trait_tys` does fulfillment and +region resolution, we must provide it `assumed_wf_types` so that we can prove +region obligations with the same expected implied bounds as +`compare_method_predicate_entailment` does. + +Since the return type of a method is understood to be one of the assumed WF +types, and we eagerly fold the return type with inference variables to do +opaque type inference, after opaque type inference, the return type will +resolve to contain the hidden types of the RPITITs. this would mean that the +hidden types of the RPITITs would be assumed to be well-formed without having +independently proven that they are. This resulted in a +[subtle unsoundness bug](https://github.com/rust-lang/rust/pull/116072). In +order to prevent this cyclic reasoning, we instead replace the hidden types of +the RPITITs in the return type of the method with *placeholders*, which lead +to no implied well-formedness bounds. + +#### Default trait body + +Type-checking a default trait body, like: + +```rust +trait Foo { + fn bar() -> impl Sized { + 1i32 + } +} +``` + +requires one interesting hack. We need to install a projection predicate +into the param-env of `Foo::bar` allowing us to assume that the RPITIT's +GAT normalizes to the RPITIT's opaque type. This relies on the +observation that a trait method and RPITIT's GAT will always be "in +sync". That is, one will only ever be overridden if the other one is as +well. + +Compare this to a similar desugaring of the code above, which would fail +because we cannot rely on this same assumption: + +```rust +#![feature(impl_trait_in_assoc_type)] +#![feature(associated_type_defaults)] + +trait Foo { + type RPITIT = impl Sized; + + fn bar() -> Self::RPITIT { + 01i32 + } +} +``` + +Failing because a down-stream impl could theoretically provide an +implementation for `RPITIT` without providing an implementation of +`foo`: + +```text +error[E0308]: mismatched types +--> src/lib.rs:8:9 + | +5 | type RPITIT = impl Sized; + | ------------------------- associated type defaults can't be assumed inside the trait defining them +6 | +7 | fn bar() -> Self::RPITIT { + | ------------ expected `::RPITIT` because of return type +8 | 01i32 + | ^^^^^ expected associated type, found `i32` + | + = note: expected associated type `::RPITIT` + found type `i32` +``` + +#### Well-formedness checking + +We check well-formedness of RPITITs just like regular associated types. + +Since we added lifetime bounds in `predicates_of` that link the +duplicated early-bound lifetimes to their original lifetimes, and we +implemented `assumed_wf_types` which inherits the WF types of the method +from which the RPITIT originates ([#113704]), we have no issues +WF-checking the GAT as if it were a regular GAT. + +### What's broken, what's weird, etc. + +##### Specialization is super busted + +The "default trait methods" described above does not interact well with +specialization, because we only install those projection bounds in trait +default methods, and not in impl methods. Given that specialization is +already pretty busted, I won't go into detail, but it's currently a bug +tracked in: + * `tests/ui/impl-trait/in-trait/specialization-broken.rs` + +##### Projections don't have variances + +This code fails because projections don't have variances: +```rust +#![feature(return_position_impl_trait_in_trait)] + +trait Foo { + // Note that the RPITIT below does *not* capture `'lt`. + fn bar<'lt: 'lt>() -> impl Eq; +} + +fn test<'a, 'b, T: Foo>() -> bool { + ::bar::<'a>() == ::bar::<'b>() + //~^ ERROR + // (requires that `'a == 'b`) +} +``` + +This is because we can't relate `::Rpitit<'a>` and `::Rpitit<'b>`, even if they don't capture their lifetime. If we were +using regular opaque types, this would work, because they would be +bivariant in that lifetime parameter: +```rust +#![feature(return_position_impl_trait_in_trait)] + +fn bar<'lt: 'lt>() -> impl Eq { + () +} + +fn test<'a, 'b>() -> bool { + bar::<'a>() == bar::<'b>() +} +``` + +This is probably okay though, since RPITITs will likely have their +captures behavior changed to capture all in-scope lifetimes anyways. +This could also be relaxed later in a forwards-compatible way if we were +to consider variances of RPITITs when relating projections. + +[#112988]: https://github.com/rust-lang/rust/pull/112988 +[RFC 3425]: https://github.com/rust-lang/rfcs/pull/3425 +[#101224]: https://github.com/rust-lang/rust/pull/101224 +[#113704]: https://github.com/rust-lang/rust/pull/113704 diff --git a/src/rustbot.md b/src/rustbot.md index 5350f44ef..0959224f0 100644 --- a/src/rustbot.md +++ b/src/rustbot.md @@ -44,13 +44,13 @@ the `@rustbot` command will look like this: @rustbot label -S-waiting-on-author +S-waiting-on-review The syntax for this command is pretty loose, so there are other variants of this -command invocation. For more details, see [the wiki page about labeling][labeling]. +command invocation. For more details, see [the docs page about labeling][labeling]. -[labeling]: https://github.com/rust-lang/triagebot/wiki/Labeling +[labeling]: https://forge.rust-lang.org/triagebot/labeling.html ## Other commands -If you are interested in seeing what `@rustbot` is capable of, check out its [wiki], +If you are interested in seeing what `@rustbot` is capable of, check out its [documentation], which is meant as a reference for the bot and should be kept up to date every time the bot gets an upgrade. @@ -58,6 +58,6 @@ bot gets an upgrade. existing commands or suggestions for new commands, feel free to reach out [on Zulip][zulip] or file an issue in [the triagebot repository][repo] -[wiki]: https://github.com/rust-lang/triagebot/wiki +[documentation]: https://forge.rust-lang.org/triagebot/index.html [zulip]: https://rust-lang.zulipchat.com/#narrow/stream/224082-t-release.2Ftriagebot [repo]: https://github.com/rust-lang/triagebot/ diff --git a/src/rustc-driver-getting-diagnostics.md b/src/rustc-driver-getting-diagnostics.md index 5ce93c3df..95e3c7cc8 100644 --- a/src/rustc-driver-getting-diagnostics.md +++ b/src/rustc-driver-getting-diagnostics.md @@ -7,38 +7,8 @@ To get diagnostics from the compiler, configure `rustc_interface::Config` to output diagnostic to a buffer, and run `TyCtxt.analysis`. The following was tested -with `nightly-2022-06-05` (See [here][example] -for the complete example): - -[example]: https://github.com/rust-lang/rustc-dev-guide/blob/master/examples/rustc-driver-getting-diagnostics.rs +with `nightly-2023-10-03`: ```rust -let buffer = sync::Arc::new(sync::Mutex::new(Vec::new())); -let config = rustc_interface::Config { - opts: config::Options { - // Configure the compiler to emit diagnostics in compact JSON format. - error_format: config::ErrorOutputType::Json { - pretty: false, - json_rendered: rustc_errors::emitter::HumanReadableErrorType::Default( - rustc_errors::emitter::ColorConfig::Never, - ), - }, - /* other config */ - }, - // Redirect the diagnostic output of the compiler to a buffer. - diagnostic_output: rustc_session::DiagnosticOutput::Raw(Box::from(DiagnosticSink( - buffer.clone(), - ))), - /* other config */ -}; -rustc_interface::run_compiler(config, |compiler| { - compiler.enter(|queries| { - queries.global_ctxt().unwrap().take().enter(|tcx| { - // Run the analysis phase on the local crate to trigger the type error. - let _ = tcx.analysis(()); - }); - }); -}); -// Read buffered diagnostics. -let diagnostics = String::from_utf8(buffer.lock().unwrap().clone()).unwrap(); +{{#include ../examples/rustc-driver-getting-diagnostics.rs}} ``` diff --git a/src/rustc-driver-interacting-with-the-ast.md b/src/rustc-driver-interacting-with-the-ast.md index ce53f3861..fc119c1ec 100644 --- a/src/rustc-driver-interacting-with-the-ast.md +++ b/src/rustc-driver-interacting-with-the-ast.md @@ -5,45 +5,8 @@ ## Getting the type of an expression To get the type of an expression, use the `global_ctxt` to get a `TyCtxt`. -The following was tested with `nightly-2022-06-05` -(see [here][example] for the complete example): - -[example]: https://github.com/rust-lang/rustc-dev-guide/blob/master/examples/rustc-driver-interacting-with-the-ast.rs +The following was tested with `nightly-2023-10-03`: ```rust -let config = rustc_interface::Config { - input: config::Input::Str { - name: source_map::FileName::Custom("main.rs".to_string()), - input: "fn main() { let message = \"Hello, world!\"; println!(\"{}\", message); }" - .to_string(), - }, - /* other config */ -}; -rustc_interface::run_compiler(config, |compiler| { - compiler.enter(|queries| { - // Analyze the crate and inspect the types under the cursor. - queries.global_ctxt().unwrap().take().enter(|tcx| { - // Every compilation contains a single crate. - let hir_krate = tcx.hir(); - // Iterate over the top-level items in the crate, looking for the main function. - for id in hir_krate.items() { - let item = hir_krate.item(id); - // Use pattern-matching to find a specific node inside the main function. - if let rustc_hir::ItemKind::Fn(_, _, body_id) = item.kind { - let expr = &tcx.hir().body(body_id).value; - if let rustc_hir::ExprKind::Block(block, _) = expr.kind { - if let rustc_hir::StmtKind::Local(local) = block.stmts[0].kind { - if let Some(expr) = local.init { - let hir_id = expr.hir_id; // hir_id identifies the string "Hello, world!" - let def_id = tcx.hir().local_def_id(item.hir_id()); // def_id identifies the main function - let ty = tcx.typeck(def_id).node_type(hir_id); - println!("{:?}: {:?}", expr, ty); - } - } - } - } - } - }) - }); -}); +{{#include ../examples/rustc-driver-interacting-with-the-ast.rs}} ``` diff --git a/src/rustc-driver.md b/src/rustc-driver.md index cef50111d..bae98c746 100644 --- a/src/rustc-driver.md +++ b/src/rustc-driver.md @@ -1,4 +1,4 @@ -# The Rustc Driver and Interface +# `rustc_driver` and `rustc_interface` The [`rustc_driver`] is essentially `rustc`'s `main()` function. It acts as the glue for running the various phases of the compiler in the correct order, @@ -32,7 +32,7 @@ as well as allowing some custom code run after different phases of the compilati [cb]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/trait.Callbacks.html -[rd_rc]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver/fn.run_compiler.html +[rd_rc]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_driver_impl/fn.run_compiler.html [i_rc]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/interface/fn.run_compiler.html [example]: https://github.com/rust-lang/rustc-dev-guide/blob/master/examples/rustc-driver-example.rs [`rustc_interface`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_interface/index.html diff --git a/src/rustdoc-internals.md b/src/rustdoc-internals.md index c85e82e96..c35f01549 100644 --- a/src/rustdoc-internals.md +++ b/src/rustdoc-internals.md @@ -7,38 +7,61 @@ see the ["Rustdoc overview" chapter](./rustdoc.md). ## From crate to clean -In `core.rs` are two central items: the `DocContext` struct, and the `run_core` -function. The latter is where rustdoc calls out to rustc to compile a crate to -the point where rustdoc can take over. The former is a state container used -when crawling through a crate to gather its documentation. +In `core.rs` are two central items: the `DocContext` struct, and the +`run_global_ctxt` function. The latter is where rustdoc calls out to rustc to +compile a crate to the point where rustdoc can take over. The former is a state +container used when crawling through a crate to gather its documentation. The main process of crate crawling is done in `clean/mod.rs` through several -implementations of the `Clean` trait defined within. This is a conversion -trait, which defines one method: +functions with names that start with `clean_`. Each function accepts an `hir` +or `ty` data structure, and outputs a `clean` structure used by rustdoc. For +example, this function for converting lifetimes: ```rust,ignore -pub trait Clean { - fn clean(&self, cx: &DocContext) -> T; +fn clean_lifetime<'tcx>(lifetime: &hir::Lifetime, cx: &mut DocContext<'tcx>) -> Lifetime { + let def = cx.tcx.named_bound_var(lifetime.hir_id); + if let Some( + rbv::ResolvedArg::EarlyBound(node_id) + | rbv::ResolvedArg::LateBound(_, _, node_id) + | rbv::ResolvedArg::Free(_, node_id), + ) = def + { + if let Some(lt) = cx.args.get(&node_id).and_then(|p| p.as_lt()).cloned() { + return lt; + } + } + Lifetime(lifetime.ident.name) } ``` `clean/mod.rs` also defines the types for the "cleaned" AST used later on to -render documentation pages. Each usually accompanies an implementation of -`Clean` that takes some AST or HIR type from rustc and converts it into the +render documentation pages. Each usually accompanies a `clean` function +that takes some AST or HIR type from rustc and converts it into the appropriate "cleaned" type. "Big" items like modules or associated items may -have some extra processing in its `Clean` implementation, but for the most part +have some extra processing in its `clean` function, but for the most part these impls are straightforward conversions. The "entry point" to this module -is the `impl Clean for visit_ast::RustdocVisitor`, which is called by -`run_core` above. - -You see, I actually lied a little earlier: There's another AST transformation -that happens before the events in `clean/mod.rs`. In `visit_ast.rs` is the -type `RustdocVisitor`, which *actually* crawls a `rustc_hir::Crate` to get the first -intermediate representation, defined in `doctree.rs`. This pass is mainly to -get a few intermediate wrappers around the HIR types and to process visibility -and inlining. This is where `#[doc(inline)]`, `#[doc(no_inline)]`, and -`#[doc(hidden)]` are processed, as well as the logic for whether a `pub use` -should get the full page or a "Reexport" line in the module page. +is `clean::krate`, which is called by +`run_global_ctxt` above. + +The first step in `clean::krate` is to invoke `visit_ast::RustdocVisitor` to +process the module tree into an intermediate `visit_ast::Module`. This is the +step that actually crawls the `rustc_hir::Crate`, normalizing various aspects +of name resolution, such as: + + * showing `#[macro_export]`-ed macros at the crate root, regardless of where + they're defined + * inlining public `use` exports of private items, or showing a "Reexport" + line in the module page + * inlining items with `#[doc(hidden)]` if the base item is hidden but the + reexport is not + * handling `#[doc(inline)]` and `#[doc(no_inline)]` + * handling import globs and cycles, so there are no duplicates or infinite + directory trees + +After this step, `clean::krate` invokes `clean_doc_module`, which actually +converts the HIR items to the cleaned AST. This is also the step where cross- +crate inlining is performed, which requires converting `rustc_middle` data +structures into the cleaned AST instead. The other major thing that happens in `clean/mod.rs` is the collection of doc comments and `#[doc=""]` attributes into a separate field of the Attributes @@ -48,41 +71,28 @@ easier to collect this documentation later in the process. The primary output of this process is a `clean::Crate` with a tree of Items which describe the publicly-documentable items in the target crate. -### Hot potato +### Passes anything but a gas station + +(alternate title: [hot potato](https://www.youtube.com/watch?v=WNFBIt5HxdY)) Before moving on to the next major step, a few important "passes" occur over -the documentation. These do things like combine the separate "attributes" into -a single string to make the document easier on the markdown parser, -or drop items that are not public or deliberately hidden with `#[doc(hidden)]`. +the cleaned AST. Several of these passes are lints and reports, but some of +them mutate or generate new items. + These are all implemented in the `passes/` directory, one file per pass. By default, all of these passes are run on a crate, but the ones regarding dropping private/hidden items can be bypassed by passing `--document-private-items` to rustdoc. Note that unlike the previous set of AST transformations, the passes are run on the _cleaned_ crate. -(Strictly speaking, you can fine-tune the passes run and even add your own, but -[we're trying to deprecate that][44136]. If you need finer-grain control over -these passes, please let us know!) - -[44136]: https://github.com/rust-lang/rust/issues/44136 - -Here is the list of passes as of November 2022: +Here is the list of passes as of March 2023: - `calculate-doc-coverage` calculates information used for the `--show-coverage` flag. -- `check-bare-urls` detects links that are not linkified, e.g., in Markdown such as - `Go to https://example.com/.` It suggests wrapping the link with angle brackets: - `Go to .` to linkify it. This is the code behind the `rustdoc::bare_urls` lint. - -- `check-code-block-syntax` validates syntax inside Rust code blocks - (```rust) - -- `check-doc-test-visibility` runs doctest visibility–related lints. - -- `check-invalid-html-tags` detects invalid HTML (like an unclosed ``) - in doc comments. +- `check-doc-test-visibility` runs doctest visibility–related lints. This pass + runs before `strip-private`, which is why it needs to be separate from + `run-lints`. - `collect-intra-doc-links` resolves [intra-doc links](https://doc.rust-lang.org/nightly/rustdoc/write-documentation/linking-to-items-by-name.html). @@ -92,44 +102,66 @@ Here is the list of passes as of November 2022: - `propagate-doc-cfg` propagates `#[doc(cfg(...))]` to child items. +- `run-lints` runs some of rustdoc's lints, defined in `passes/lint`. This is + the last pass to run. + + - `bare_urls` detects links that are not linkified, e.g., in Markdown such as + `Go to https://example.com/.` It suggests wrapping the link with angle brackets: + `Go to .` to linkify it. This is the code behind the `rustdoc::bare_urls` lint. + + - `check_code_block_syntax` validates syntax inside Rust code blocks + (```rust) + + - `html_tags` detects invalid HTML (like an unclosed ``) + in doc comments. + +- `strip-hidden` and `strip-private` strip all `doc(hidden)` and private items + from the output. `strip-private` implies `strip-priv-imports`. Basically, the + goal is to remove items that are not relevant for public documentation. This + pass is skipped when `--document-hidden-items` is passed. + - `strip-priv-imports` strips all private import statements (`use`, `extern crate`) from a crate. This is necessary because rustdoc will handle *public* imports by either inlining the item's documentation to the module or creating a "Reexports" section with the import in it. The pass ensures that all of - these imports are actually relevant to documentation. + these imports are actually relevant to documentation. It is technically + only run when `--document-private-items` is passed, but `strip-private` + accomplishes the same thing. -- `strip-hidden` and `strip-private` strip all `doc(hidden)` and private items - from the output. `strip-private` implies `strip-priv-imports`. Basically, the - goal is to remove items that are not relevant for public documentation. +- `strip-private` strips all private items from a crate which cannot be seen + externally. This pass is skipped when `--document-private-items` is passed. There is also a `stripper` module in `passes/`, but it is a collection of utility functions for the `strip-*` passes and is not a pass itself. -## From clean to crate +## From clean to HTML This is where the "second phase" in rustdoc begins. This phase primarily lives -in the `html/` folder, and it all starts with `run()` in `html/render.rs`. This -code is responsible for setting up the `Context`, `SharedContext`, and `Cache` -which are used during rendering, copying out the static files which live in -every rendered set of documentation (things like the fonts, CSS, and JavaScript -that live in `html/static/`), creating the search index, and printing out the -source code rendering, before beginning the process of rendering all the -documentation for the crate. - -Several functions implemented directly on `Context` take the `clean::Crate` and -set up some state between rendering items or recursing on a module's child -items. From here the "page rendering" begins, via an enormous `write!()` call -in `html/layout.rs`. The parts that actually generate HTML from the items and -documentation occurs within a series of `std::fmt::Display` implementations and -functions that pass around a `&mut std::fmt::Formatter`. The top-level -implementation that writes out the page body is the `impl<'a> fmt::Display for -Item<'a>` in `html/render.rs`, which switches out to one of several `item_*` -functions based on the kind of `Item` being rendered. +in the `formats/` and `html/` folders, and it all starts with +`formats::run_format`. This code is responsible for setting up a type that +`impl FormatRenderer`, which for HTML is [`Context`]. + +This structure contains methods that get called by `run_format` to drive the +doc rendering, which includes: + +* `init` generates `static.files`, as well as search index and `src/` +* `item` generates the item HTML files themselves +* `after_krate` generates other global resources like `all.html` + +In `item`, the "page rendering" occurs, via a mixture of [Askama] templates +and manual `write!()` calls, starting in `html/layout.rs`. The parts that have +not been converted to templates occur within a series of `std::fmt::Display` +implementations and functions that pass around a `&mut std::fmt::Formatter`. + +The parts that actually generate HTML from the items and documentation start +with `print_item` defined in `html/render/print_item.rs`, which switches out +to one of several `item_*` functions based on kind of `Item` being rendered. Depending on what kind of rendering code you're looking for, you'll probably -find it either in `html/render.rs` for major items like "what sections should I -print for a struct page" or `html/format.rs` for smaller component pieces like -"how should I print a where clause as part of some other item". +find it either in `html/render/mod.rs` for major items like "what sections +should I print for a struct page" or `html/format/mod.rs` for smaller +component pieces like "how should I print a where clause as part of some other item". Whenever rustdoc comes across an item that should print hand-written documentation alongside, it calls out to `html/markdown.rs` which interfaces @@ -141,6 +173,8 @@ running the Markdown parser. There's also a function in here (`find_testable_code`) that specifically scans for Rust code blocks so the test-runner code can find all the doctests in the crate. +[Askama]: https://docs.rs/askama/latest/askama/ + ### From soup to nuts (alternate title: ["An unbroken thread that stretches from those first `Cell`s @@ -148,23 +182,46 @@ to us"][video]) [video]: https://www.youtube.com/watch?v=hOLAGYmUQV0 -It's important to note that the AST cleaning can ask the compiler for -information (crucially, `DocContext` contains a `TyCtxt`), but page rendering -cannot. The `clean::Crate` created within `run_core` is passed outside the -compiler context before being handed to `html::render::run`. This means that a -lot of the "supplementary data" that isn't immediately available inside an -item's definition, like which trait is the `Deref` trait used by the language, -needs to be collected during cleaning, stored in the `DocContext`, and passed -along to the `SharedContext` during HTML rendering. This manifests as a bunch -of shared state, context variables, and `RefCell`s. - -Also of note is that some items that come from "asking the compiler" don't go -directly into the `DocContext` - for example, when loading items from a foreign -crate, rustdoc will ask about trait implementations and generate new `Item`s -for the impls based on that information. This goes directly into the returned -`Crate` rather than roundabout through the `DocContext`. This way, these -implementations can be collected alongside the others, right before rendering -the HTML. +It's important to note that rustdoc can ask the compiler for type information +directly, even during HTML generation. This [didn't used to be the case], and +a lot of rustdoc's architecture was designed around not doing that, but a +`TyCtxt` is now passed to `formats::renderer::run_format`, which is used to +run generation for both HTML and the +(unstable as of March 2023) JSON format. + +[didn't used to be the case]: https://github.com/rust-lang/rust/pull/80090 + +This change has allowed other changes to remove data from the "clean" AST +that can be easily derived from `TyCtxt` queries, and we'll usually accept +PRs that remove fields from "clean" (it's been soft-deprecated), but this +is complicated from two other constraints that rustdoc runs under: + +* Docs can be generated for crates that don't actually pass type checking. + This is used for generating docs that cover mutually-exclusive platform + configurations, such as `libstd` having a single package of docs that + cover all supported operating systems. This means rustdoc has to be able + to generate docs from HIR. +* Docs can inline across crates. Since crate metadata doesn't contain HIR, + it must be possible to generate inlined docs from the `rustc_middle` data. + +The "clean" AST acts as a common output format for both input formats. There +is also some data in clean that doesn't correspond directly to HIR, such as +synthetic `impl`s for auto traits and blanket `impl`s generated by the +`collect-trait-impls` pass. + +Some additional data is stored in +`html::render::context::{Context, SharedContext}`. These two types serve as +ways to segregate rustdoc's data for an eventual future with multithreaded doc +generation, as well as just keeping things organized: + +* [`Context`] stores data used for generating the current page, such as its + path, a list of HTML IDs that have been used (to avoid duplicate `id=""`), + and the pointer to `SharedContext`. +* [`SharedContext`] stores data that does not vary by page, such as the `tcx` + pointer, and a list of all types. + +[`Context`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/html/render/context/struct.Context.html +[`SharedContext`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustdoc/html/render/context/struct.SharedContext.html ## Other tricks up its sleeve @@ -191,7 +248,7 @@ Some extra reading about `make_test` can be found So that's rustdoc's code in a nutshell, but there's more things in the repo that deal with it. Since we have the full `compiletest` suite at hand, there's -a set of tests in `src/test/rustdoc` that make sure the final HTML is what we +a set of tests in `tests/rustdoc` that make sure the final HTML is what we expect in various situations. These tests also use a supplementary script, `src/etc/htmldocck.py`, that allows it to look through the final HTML using XPath notation to get a precise look at the output. The full description of all @@ -204,7 +261,7 @@ directory relative to the test file with the comment. If you need to build docs for the auxiliary file, use `// build-aux-docs`. In addition, there are separate tests for the search index and rustdoc's -ability to query it. The files in `src/test/rustdoc-js` each contain a +ability to query it. The files in `tests/rustdoc-js` each contain a different search query and the expected results, broken out by search tab. These files are processed by a script in `src/tools/rustdoc-js` and the Node.js runtime. These tests don't have as thorough of a writeup, but a broad example @@ -224,7 +281,7 @@ server. To test these features locally, you can run a local HTTP server, like this: ```bash -$ ./x.py doc library +$ ./x doc library # The documentation has been generated into `build/[YOUR ARCH]/doc`. $ python3 -m http.server -d build/[YOUR ARCH]/doc ``` diff --git a/src/rustdoc.md b/src/rustdoc.md index 23428efd6..02da8be81 100644 --- a/src/rustdoc.md +++ b/src/rustdoc.md @@ -1,8 +1,5 @@ # Rustdoc overview -`rustdoc` uses `rustc` internals (and, of course, the standard library), so you -will have to build the compiler and `std` once before you can build `rustdoc`. - `rustdoc` lives in-tree with the compiler and standard library. This chapter is about how it works. For information about Rustdoc's features and how to use them, see @@ -12,6 +9,11 @@ For more details about how rustdoc works, see the [Rustdoc internals]: ./rustdoc-internals.md + + +`rustdoc` uses `rustc` internals (and, of course, the standard library), so you +will have to build the compiler and `std` once before you can build `rustdoc`. + Rustdoc is implemented entirely within the crate [`librustdoc`][rd]. It runs the compiler up to the point where we have an internal representation of a crate (HIR) and the ability to run some queries about the types of items. [HIR] @@ -39,24 +41,24 @@ does is call the `main()` that's in this crate's `lib.rs`, though.) ## Cheat sheet -* Run `./x.py setup tools` before getting started. This will configure `x.py` +* Run `./x setup tools` before getting started. This will configure `x` with nice settings for developing rustdoc and other tools, including downloading a copy of rustc rather than building it. -* Use `./x.py check src/tools/rustdoc` to quickly check for compile errors. -* Use `./x.py build` to make a usable +* Use `./x check src/tools/rustdoc` to quickly check for compile errors. +* Use `./x build` to make a usable rustdoc you can run on other projects. * Add `library/test` to be able to use `rustdoc --test`. - * Run `rustup toolchain link stage2 build/$TARGET/stage2` to add a + * Run `rustup toolchain link stage2 build/host/stage2` to add a custom toolchain called `stage2` to your rustup environment. After running that, `cargo +stage2 doc` in any directory will build with your locally-compiled rustdoc. -* Use `./x.py doc library` to use this rustdoc to generate the +* Use `./x doc library` to use this rustdoc to generate the standard library docs. - * The completed docs will be available in `build/$TARGET/doc` (under `core`, `alloc`, and `std`). + * The completed docs will be available in `build/host/doc` (under `core`, `alloc`, and `std`). * If you want to copy those docs to a webserver, copy all of - `build/$TARGET/doc`, since that's where the CSS, JS, fonts, and landing + `build/host/doc`, since that's where the CSS, JS, fonts, and landing page are. -* Use `./x.py test src/test/rustdoc*` to run the tests using a stage1 +* Use `./x test tests/rustdoc*` to run the tests using a stage1 rustdoc. * See [Rustdoc internals] for more information about tests. @@ -72,13 +74,13 @@ does is call the `main()` that's in this crate's `lib.rs`, though.) `doctest.rs`. * The Markdown renderer is loaded up in `html/markdown.rs`, including functions for extracting doctests from a given block of Markdown. -* The tests on the structure of rustdoc HTML output are located in `src/test/rustdoc`, where +* The tests on the structure of rustdoc HTML output are located in `tests/rustdoc`, where they're handled by the test runner of rustbuild and the supplementary script `src/etc/htmldocck.py`. ## Tests -* All paths in this section are relative to `src/test` in the rust-lang/rust repository. +* All paths in this section are relative to `tests` in the rust-lang/rust repository. * Tests on search index generation are located in `rustdoc-js`, as a series of JavaScript files that encode queries on the standard library search index and expected results. diff --git a/src/salsa.md b/src/salsa.md index 66f9d7479..47442ae14 100644 --- a/src/salsa.md +++ b/src/salsa.md @@ -148,9 +148,9 @@ this one depends on by specifying them as supertraits, as seen in the following example: ```rust,ignore -/// This query group is going to contain queries that depend on derived values a +/// This query group is going to contain queries that depend on derived values. A /// query group can access another query group's queries by specifying the -/// dependency as a super trait query groups can be stacked as much as needed using +/// dependency as a super trait. Query groups can be stacked as much as needed using /// that pattern. #[salsa::query_group(ParserStorage)] pub trait Parser: Inputs { @@ -168,7 +168,7 @@ belongs to, in addition to the other keys. ```rust,ignore ///This is going to be the definition of the `ast` query in the `Parser` trait. ///So, when the query `ast` is invoked, and it needs to be recomputed, Salsa is going to call this function -///and it's is going to give it the database as `impl Parser`. +///and it's going to give it the database as `impl Parser`. ///The function doesn't need to be aware of all the queries of all the query groups fn ast(db: &impl Parser, name: String) -> String { //! Note, `impl Parser` is used here but `dyn Parser` works just as well diff --git a/src/sanitizers.md b/src/sanitizers.md index e3aa44ea9..27d40a11a 100644 --- a/src/sanitizers.md +++ b/src/sanitizers.md @@ -76,8 +76,8 @@ implementation: ## Testing sanitizers Sanitizers are validated by code generation tests in -[`src/test/codegen/sanitize*.rs`][test-cg] and end-to-end functional tests in -[`src/test/ui/sanitize/`][test-ui] directory. +[`tests/codegen/sanitize*.rs`][test-cg] and end-to-end functional tests in +[`tests/ui/sanitize/`][test-ui] directory. Testing sanitizer functionality requires the sanitizer runtimes (built when `sanitizer = true` in `config.toml`) and target providing support for particular @@ -85,8 +85,8 @@ sanitizer. When sanitizer is unsupported on given target, sanitizers tests will be ignored. This behaviour is controlled by compiletest `needs-sanitizer-*` directives. -[test-cg]: https://github.com/rust-lang/rust/tree/master/src/test/codegen -[test-ui]: https://github.com/rust-lang/rust/tree/master/src/test/ui/sanitize +[test-cg]: https://github.com/rust-lang/rust/tree/master/tests/codegen +[test-ui]: https://github.com/rust-lang/rust/tree/master/tests/ui/sanitize ## Enabling sanitizer on a new target @@ -98,7 +98,7 @@ To enable a sanitizer on a new target which is already supported by LLVM: 2. [Build the runtime for the target and include it in the libdir.][sanitizer-targets] 3. [Teach compiletest that your target now supports the sanitizer.][compiletest-definition] Tests marked with `needs-sanitizer-*` should now run on the target. -4. Run tests `./x.py test --force-rerun src/test/ui/sanitize/` to verify. +4. Run tests `./x test --force-rerun tests/ui/sanitize/` to verify. 5. [--enable-sanitizers in the CI configuration][ci-configuration] to build and distribute the sanitizer runtime as part of the release process. diff --git a/src/serialization.md b/src/serialization.md index 5d8434b55..c79628106 100644 --- a/src/serialization.md +++ b/src/serialization.md @@ -108,8 +108,8 @@ and `Encodable`. - `MetadataEncodable` and `MetadataDecodable` generate implementations that only allow decoding by [`rustc_metadata::rmeta::encoder::EncodeContext`] and [`rustc_metadata::rmeta::decoder::DecodeContext`]. These are used for types - that contain `rustc_metadata::rmeta::Lazy`. -- `TyEncodable` and `TyDecoder` generate implementation that apply to any + that contain `rustc_metadata::rmeta::Lazy*`. +- `TyEncodable` and `TyDecodable` generate implementation that apply to any `TyEncoder` or `TyDecoder`. These should be used for types that are only serialized in crate metadata and/or the incremental cache, which is most serializable types in `rustc_middle`. @@ -130,7 +130,7 @@ some deserialization needs to be deferred from the initial loading of metadata. The [`LazyValue`] type wraps the (relative) offset in the crate metadata where a `T` has been serialized. There are also some variants, [`LazyArray`] and [`LazyTable`]. -The `Lazy<[T]>` and `LazyTable` type provide some functionality over +The `LazyArray<[T]>` and `LazyTable` types provide some functionality over `Lazy>` and `Lazy>`: - It's possible to encode a `LazyArray` directly from an iterator, without @@ -162,5 +162,5 @@ for `Encodable`. [`rustc_metadata::rmeta::decoder::DecodeContext`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/rmeta/decoder/struct.DecodeContext.html [`rustc_metadata::rmeta::encoder::EncodeContext`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_metadata/rmeta/encoder/struct.EncodeContext.html [`rustc_serialize`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_serialize/index.html -[`TyDecoder`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/codec/trait.TyEncoder.html -[`TyEncoder`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/codec/trait.TyDecoder.html +[`TyDecoder`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/codec/trait.TyDecoder.html +[`TyEncoder`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/codec/trait.TyEncoder.html diff --git a/src/solve/canonicalization.md b/src/solve/canonicalization.md new file mode 100644 index 000000000..a14be5216 --- /dev/null +++ b/src/solve/canonicalization.md @@ -0,0 +1,84 @@ +# Canonicalization + +Canonicalization is the process of *isolating* a value from its context and is necessary +for global caching of goals which include inference variables. + +The idea is that given the goals `u32: Trait` and `u32: Trait`, where `?x` and `?y` +are two different currently unconstrained inference variables, we should get the same result +for both goals. We can therefore prove *the canonical query* `exists u32: Trait` once +and reuse the result. + +Let's first go over the way canonical queries work and then dive into the specifics of +how canonicalization works. + +## A walkthrough of canonical queries + +To make this a bit easier, let's use the trait goal `u32: Trait` as an example with the +assumption that the only relevant impl is `impl Trait> for u32`. + +### Canonicalizing the input + +We start by *canonicalizing* the goal, replacing inference variables with existential and +placeholders with universal bound variables. This would result in the *canonical goal* +`exists u32: Trait`. + +We remember the original values of all bound variables in the original context. Here this would +map `T` back to `?x`. These original values are used later on when dealing with the query +response. + +We now call the canonical query with the canonical goal. + +### Instantiating the canonical goal inside of the query + +To actually try to prove the canonical goal we start by instantiating the bound variables with +inference variables and placeholders again. + +This happens inside of the query in a completely separate `InferCtxt`. Inside of the query we +now have a goal `u32: Trait`. We also remember which value we've used to instantiate the bound +variables in the canonical goal, which maps `T` to `?0`. + +We now compute the goal `u32: Trait` and figure out that this holds, but we've constrained +`?0` to `Vec`. We finally convert this result to something useful to the caller. + +### Canonicalizing the query response + +We have to return to the caller both whether the goal holds, and the inference constraints +from inside of the query. + +To return the inference results to the caller we canonicalize the mapping from bound variables +to the instantiated values in the query. This means that the query response is `Certainty::Yes` +and a mapping from `T` to `exists Vec`. + +### Instantiating the query response + +The caller now has to apply the constraints returned by the query. For this they first +instantiate the bound variables of the canonical response with inference variables and +placeholders again, so the mapping in the response is now from `T` to `Vec`. + +It now equates the original value of `T` (`?x`) with the value for `T` in the +response (`Vec`), which correctly constrains `?x` to `Vec`. + +## `ExternalConstraints` + +Computing a trait goal may not only constrain inference variables, it can also add region +obligations, e.g. given a goal `(): AOutlivesB<'a, 'b>` we would like to return the fact that +`'a: 'b` has to hold. + +This is done by not only returning the mapping from bound variables to the instantiated values +from the query but also extracting additional `ExternalConstraints` from the `InferCtxt` context +while building the response. + +## How exactly does canonicalization work + +TODO: link to code once the PR lands and elaborate + +- types and consts: infer to existentially bound var, placeholder to universally bound var, + considering universes +- generic parameters in the input get treated as placeholders in the root universe +- all regions in the input get all mapped to existentially bound vars and we "uniquify" them. + `&'a (): Trait<'a>` gets canonicalized to `exists<'0, '1> &'0 (): Trait<'1>`. We do not care + about their universes and simply put all regions into the highest universe of the input. +- once we collected all canonical vars we compress their universes, see comment in `finalize`. +- in the output everything in a universe of the caller gets put into the root universe and only + gets its correct universe when we unify the var values with the orig values of the caller +- we do not uniquify regions in the response and don't canonicalize `'static` \ No newline at end of file diff --git a/src/solve/coinduction.md b/src/solve/coinduction.md new file mode 100644 index 000000000..c682e002d --- /dev/null +++ b/src/solve/coinduction.md @@ -0,0 +1,250 @@ +# Coinduction + +The trait solver may use coinduction when proving goals. +Coinduction is fairly subtle so we're giving it its own chapter. + +## Coinduction and induction + +With induction, we recursively apply proofs until we end up with a finite proof tree. +Consider the example of `Vec>>: Debug` which results in the following tree. + +- `Vec>>: Debug` + - `Vec>: Debug` + - `Vec: Debug` + - `u32: Debug` + +This tree is finite. But not all goals we would want to hold have finite proof trees, +consider the following example: + +```rust +struct List { + value: T, + next: Option>>, +} +``` + +For `List: Send` to hold all its fields have to recursively implement `Send` as well. +This would result in the following proof tree: + +- `List: Send` + - `T: Send` + - `Option>>: Send` + - `Box>: Send` + - `List: Send` + - `T: Send` + - `Option>>: Send` + - `Box>: Send` + - ... + +This tree would be infinitely large which is exactly what coinduction is about. + +> To **inductively** prove a goal you need to provide a finite proof tree for it. +> To **coinductively** prove a goal the provided proof tree may be infinite. + +## Why is coinduction correct + +When checking whether some trait goals holds, we're asking "does there exist an `impl` +which satisfies this bound". Even if are infinite chains of nested goals, we still have a +unique `impl` which should be used. + +## How to implement coinduction + +While our implementation can not check for coinduction by trying to construct an infinite +tree as that would take infinite resources, it still makes sense to think of coinduction +from this perspective. + +As we cannot check for infinite trees, we instead search for patterns for which we know that +they would result in an infinite proof tree. The currently pattern we detect are (canonical) +cycles. If `T: Send` relies on `T: Send` then it's pretty clear that this will just go on forever. + +With cycles we have to be careful with caching. Because of canonicalization of regions and +inference variables encountering a cycle doesn't mean that we would get an infinite proof tree. +Looking at the following example: +```rust +trait Foo {} +struct Wrapper(T); + +impl Foo for Wrapper> +where + Wrapper: Foo +{} +``` +Proving `Wrapper: Foo` uses the impl `impl Foo for Wrapper>` which constrains +`?0` to `Wrapper` and then requires `Wrapper: Foo`. Due to canonicalization this would be +detected as a cycle. + +The idea to solve is to return a *provisional result* whenever we detect a cycle and repeatedly +retry goals until the *provisional result* is equal to the final result of that goal. We +start out by using `Yes` with no constraints as the result and then update it to the result of +the previous iteration whenever we have to rerun. + +TODO: elaborate here. We use the same approach as chalk for coinductive cycles. +Note that the treatment for inductive cycles currently differs by simply returning `Overflow`. +See [the relevant chapters][chalk] in the chalk book. + +[chalk]: https://rust-lang.github.io/chalk/book/recursive/inductive_cycles.html + + +## Future work + +We currently only consider auto-traits, `Sized`, and `WF`-goals to be coinductive. +In the future we pretty much intend for all goals to be coinductive. +Lets first elaborate on why allowing more coinductive proofs is even desirable. + +### Recursive data types already rely on coinduction... + +...they just tend to avoid them in the trait solver. + +```rust +enum List { + Nil, + Succ(T, Box>), +} + +impl Clone for List { + fn clone(&self) -> Self { + match self { + List::Nil => List::Nil, + List::Succ(head, tail) => List::Succ(head.clone(), tail.clone()), + } + } +} +``` + +We are using `tail.clone()` in this impl. For this we have to prove `Box>: Clone` +which requires `List: Clone` but that relies on the impl which we are currently checking. +By adding that requirement to the `where`-clauses of the impl, which is what we would +do with [perfect derive], we move that cycle into the trait solver and [get an error][ex1]. + +### Recursive data types + +We also need coinduction to reason about recursive types containing projections, +e.g. the following currently fails to compile even though it should be valid. +```rust +use std::borrow::Cow; +pub struct Foo<'a>(Cow<'a, [Foo<'a>]>); +``` +This issue has been known since at least 2015, see +[#23714](https://github.com/rust-lang/rust/issues/23714) if you want to know more. + +### Explicitly checked implied bounds + +When checking an impl, we assume that the types in the impl headers are well-formed. +This means that when using instantiating the impl we have to prove that's actually the case. +[#100051](https://github.com/rust-lang/rust/issues/100051) shows that this is not the case. +To fix this, we have to add `WF` predicates for the types in impl headers. +Without coinduction for all traits, this even breaks `core`. + +```rust +trait FromResidual {} +trait Try: FromResidual<::Residual> { + type Residual; +} + +struct Ready(T); +impl Try for Ready { + type Residual = Ready<()>; +} +impl FromResidual< as Try>::Residual> for Ready {} +``` + +When checking that the impl of `FromResidual` is well formed we get the following cycle: + +The impl is well formed if ` as Try>::Residual` and `Ready` are well formed. +- `wf( as Try>::Residual)` requires +- `Ready: Try`, which requires because of the super trait +- `Ready: FromResidual as Try>::Residual>`, **because of implied bounds on impl** +- `wf( as Try>::Residual)` :tada: **cycle** + +### Issues when extending coinduction to more goals + +There are some additional issues to keep in mind when extending coinduction. +The issues here are not relevant for the current solver. + +#### Implied super trait bounds + +Our trait system currently treats super traits, e.g. `trait Trait: SuperTrait`, +by 1) requiring that `SuperTrait` has to hold for all types which implement `Trait`, +and 2) assuming `SuperTrait` holds if `Trait` holds. + +Relying on 2) while proving 1) is unsound. This can only be observed in case of +coinductive cycles. Without cycles, whenever we rely on 2) we must have also +proven 1) without relying on 2) for the used impl of `Trait`. + +```rust +trait Trait: SuperTrait {} + +impl Trait for T {} + +// Keeping the current setup for coinduction +// would allow this compile. Uff :< +fn sup() {} +fn requires_trait() { sup::() } +fn generic() { requires_trait::() } +``` +This is not really fundamental to coinduction but rather an existing property +which is made unsound because of it. + +##### Possible solutions + +The easiest way to solve this would be to completely remove 2) and always elaborate +`T: Trait` to `T: Trait` and `T: SuperTrait` outside of the trait solver. +This would allow us to also remove 1), but as we still have to prove ordinary +`where`-bounds on traits, that's just additional work. + +While one could imagine ways to disable cyclic uses of 2) when checking 1), +at least the ideas of myself - @lcnr - are all far to complex to be reasonable. + +#### `normalizes_to` goals and progress + +A `normalizes_to` goal represents the requirement that `::Assoc` normalizes +to some `U`. This is achieved by defacto first normalizing `::Assoc` and then +equating the resulting type with `U`. It should be a mapping as each projection should normalize +to exactly one type. By simply allowing infinite proof trees, we would get the following behavior: + +```rust +trait Trait { + type Assoc; +} + +impl Trait for () { + type Assoc = <() as Trait>::Assoc; +} +``` + +If we now compute `normalizes_to(<() as Trait>::Assoc, Vec)`, we would resolve the impl +and get the associated type `<() as Trait>::Assoc`. We then equate that with the expected type, +causing us to check `normalizes_to(<() as Trait>::Assoc, Vec)` again. +This just goes on forever, resulting in an infinite proof tree. + +This means that `<() as Trait>::Assoc` would be equal to any other type which is unsound. + +##### How to solve this + +**WARNING: THIS IS SUBTLE AND MIGHT BE WRONG** + +Unlike trait goals, `normalizes_to` has to be *productive*[^1]. A `normalizes_to` goal +is productive once the projection normalizes to a rigid type constructor, +so `<() as Trait>::Assoc` normalizing to `Vec<<() as Trait>::Assoc>` would be productive. + +A `normalizes_to` goal has two kinds of nested goals. Nested requirements needed to actually +normalize the projection, and the equality between the normalized projection and the +expected type. Only the equality has to be productive. A branch in the proof tree is productive +if it is either finite, or contains at least one `normalizes_to` where the alias is resolved +to a rigid type constructor. + +Alternatively, we could simply always treat the equate branch of `normalizes_to` as inductive. +Any cycles should result in infinite types, which aren't supported anyways and would only +result in overflow when deeply normalizing for codegen. + +experimentation and examples: https://hackmd.io/-8p0AHnzSq2VAE6HE_wX-w?view + +Another attempt at a summary. +- in projection eq, we must make progress with constraining the rhs +- a cycle is only ok if while equating we have a rigid ty on the lhs after norm at least once +- cycles outside of the recursive `eq` call of `normalizes_to` are always fine + +[^1]: related: https://coq.inria.fr/refman/language/core/coinductive.html#top-level-definitions-of-corecursive-functions + +[perfect derive]: https://smallcultfollowing.com/babysteps/blog/2022/04/12/implied-bounds-and-perfect-derive +[ex1]: https://play.rust-lang.org/?version=stable&mode=debug&edition=2021&gist=0a9c3830b93a2380e6978d6328df8f72 diff --git a/src/solve/invariants.md b/src/solve/invariants.md new file mode 100644 index 000000000..75ae53070 --- /dev/null +++ b/src/solve/invariants.md @@ -0,0 +1,154 @@ +# Invariants of the type system + +FIXME: This file talks about invariants of the type system as a whole, not only the solver + +There are a lot of invariants - things the type system guarantees to be true at all times - +which are desirable or expected from other languages and type systems. Unfortunately, quite +a few of them do not hold in Rust right now. This is either a fundamental to its design or +caused by bugs and something that may change in the future. + +It is important to know about the things you can assume while working on - and with - the +type system, so here's an incomplete and inofficial list of invariants of +the core type system: + +- ✅: this invariant mostly holds, with some weird exceptions, you can rely on it outside +of these cases +- ❌: this invariant does not hold, either due to bugs or by design, you must not rely on +it for soundness or have to be incredibly careful when doing so + +### `wf(X)` implies `wf(normalize(X))` ✅ + +If a type containing aliases is well-formed, it should also be +well-formed after normalizing said aliases. We rely on this as +otherwise we would have to re-check for well-formedness for these +types. + +This is unfortunately broken for `>::Output` due to implied bounds, +resulting in [#114936]. + +### Structural equality modulo regions implies semantic equality ✅ + +If you have a some type and equate it to itself after replacing any regions with unique +inference variables in both the lhs and rhs, the now potentially structurally different +types should still be equal to each other. + +Needed to prevent goals from succeeding in HIR typeck and then failing in MIR borrowck. +If this does invariant is broken MIR typeck ends up failing with an ICE. + +### Applying inference results from a goal does not change its result ❌ + +TODO: this invariant is formulated in a weird way and needs to be elaborated. +Pretty much: I would like this check to only fail if there's a solver bug: +https://github.com/rust-lang/rust/blob/2ffeb4636b4ae376f716dc4378a7efb37632dc2d/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs#L391-L407 + +If we prove some goal/equate types/whatever, apply the resulting inference constraints, +and then redo the original action, the result should be the same. + +This unfortunately does not hold - at least in the new solver - due to a few annoying reasons. + +### The trait solver has to be *locally sound* ✅ + +This means that we must never return *success* for goals for which no `impl` exists. That would +mean we assume a trait is implemented even though it is not, which is very likely to result in +actual unsoundness. When using `where`-bounds to prove a goal, the `impl` will be provided by the +user of the item. + +This invariant only holds if we check region constraints. As we do not check region constraints +during implicit negative overlap check in coherence, this invariant is broken there. As this check +relies on *completeness* of the trait solver, it is not able to use the current region constraints +check - `InferCtxt::resolve_regions` - as its handling of type outlives goals is incomplete. + +### Normalization of semantically equal aliases in empty environments results in a unique type ✅ + +Normalization for alias types/consts has to have a unique result. Otherwise we can easily +implement transmute in safe code. Given the following function, we have to make sure that +the input and output types always get normalized to the same concrete type. + +```rust +fn foo( + x: ::Assoc +) -> ::Assoc { + x +} +``` + +Many of the currently known unsound issues end up relying on this invariant being broken. +It is however very difficult to imagine a sound type system without this invariant, so +the issue is that the invariant is broken, not that we incorrectly rely on it. + +### Generic goals and their instantiations have the same result ✅ + +Pretty much: If we successfully typecheck a generic function concrete instantiations +of that function should also typeck. We should not get errors post-monomorphization. +We can however get overflow errors at that point. + +TODO: example for overflow error post-monomorphization + +This invariant is relied on to allow the normalization of generic aliases. Breaking +it can easily result in unsoundness, e.g. [#57893](https://github.com/rust-lang/rust/issues/57893) + +### Trait goals in empty environments are proven by a unique impl ✅ + +If a trait goal holds with an empty environment, there should be a unique `impl`, +either user-defined or builtin, which is used to prove that goal. This is +necessary to select a unique method. It + +We do however break this invariant in few cases, some of which are due to bugs, +some by design: +- *marker traits* are allowed to overlap as they do not have associated items +- *specialization* allows specializing impls to overlap with their parent +- the builtin trait object trait implementation can overlap with a user-defined impl: +[#57893] + +### The type system is complete ❌ + +The type system is not complete, it often adds unnecessary inference constraints, and errors +even though the goal could hold. + +- method selection +- opaque type inference +- handling type outlives constraints +- preferring `ParamEnv` candidates over `Impl` candidates during candidate selection +in the trait solver + +#### The type system is complete during the implicit negative overlap check in coherence ✅ + +During the implicit negative overlap check in coherence we must never return *error* for +goals which can be proven. This would allow for overlapping impls with potentially different +associated items, breaking a bunch of other invariants. + +This invariant is currently broken in many different ways while actually something we rely on. +We have to be careful as it is quite easy to break: +- generalization of aliases +- generalization during subtyping binders (luckily not exploitable in coherence) + +### Trait solving must be (free) lifetime agnostic ✅ + +Trait solving during codegen should have the same result as during typeck. As we erase +all free regions during codegen we must not rely on them during typeck. A noteworthy example +is special behavior for `'static`. + +We also have to be careful with relying on equality of regions in the trait solver. +This is fine for codegen, as we treat all erased regions as equal. We can however +lose equality information from HIR to MIR typeck. + +The new solver "uniquifies regions" during canonicalization, canonicalizing `u32: Trait<'x, 'x>` +as `exists<'0, '1> u32: Trait<'0, '1>`, to make it harder to rely on this property. + +### Removing ambiguity makes strictly more things compile ❌ + +Ideally we *should* not rely on ambiguity for things to compile. +Not doing that will cause future improvements to be breaking changes. + +Due to *incompleteness* this is not the case and improving inference can result in inference +changes, breaking existing projects. + +### Semantic equality implies structural equality ✅ + +Two types being equal in the type system must mean that they have the +same `TypeId` after instantiating their generic parameters with concrete +arguments. This currently does not hold: [#97156]. + +[#57893]: https://github.com/rust-lang/rust/issues/57893 +[#97156]: https://github.com/rust-lang/rust/issues/97156 +[#114936]: https://github.com/rust-lang/rust/issues/114936 \ No newline at end of file diff --git a/src/solve/normalization.md b/src/solve/normalization.md new file mode 100644 index 000000000..653c976a4 --- /dev/null +++ b/src/solve/normalization.md @@ -0,0 +1,80 @@ +# Normalization in the new solver + +With the new solver we've made some fairly significant changes to normalization when compared +to the existing implementation. + +We now differentiate between "shallow normalization" and "deep normalization". +"Shallow normalization" normalizes a type until it is no-longer a potentially normalizeable alias; +it does not recurse into the type. "deep normalization" replaces all normalizeable aliases in a +type with its underlying type. + +The old trait solver currently always deeply normalizes via `Projection` obligations. +This is the only way to normalize in the old solver. By replacing projections with a new +inference variable and then emitting `Projection(::Assoc, ?new_infer)` the old +solver successfully deeply normalizes even in the case of ambiguity. This approach does not +work for projections referencing bound variables. + +## Inside of the trait solver + +Normalization in the new solver exclusively happens via `Projection`[^0] goals. +This only succeeds by first normalizing the alias by one level and then equating +it with the expected type. This differs from [the behavior of projection clauses] +which can also be proven by successfully equating the projection without normalizating. +This means that `Projection`[^0] goals must only be used in places where we +*have to normalize* to make progress. To normalize `::Assoc`, we first create +a fresh inference variable `?normalized` and then prove +`Projection(::Assoc, ?normalized)`[^0]. `?normalized` is then constrained to +the underlying type. + +Inside of the trait solver we never deeply normalize. we only apply shallow normalization +in [`assemble_candidates_after_normalizing_self_ty`] and inside for [`AliasRelate`] +goals for the [`normalizes-to` candidates]. + +## Outside of the trait solver + +The core type system - relating types and trait solving - will not need deep +normalization with the new solver. There are still some areas which depend on it. +For these areas there is the function `At::deeply_normalize`. Without additional +trait solver support deep normalization does not always work in case of ambiguity. +Luckily deep normalization is currently only necessary in places where there is no ambiguity. +`At::deeply_normalize` immediately fails if there's ambiguity. + +If we only care about the outermost layer of types, we instead use +`At::structurally_normalize` or `FnCtxt::(try_)structurally_resolve_type`. +Unlike `At::deeply_normalize`, shallow normalization is also used in cases where we +have to handle ambiguity. `At::structurally_normalize` normalizes until the self type +is either rigid or an inference variable and we're stuck with ambiguity. This means +that the self type may not be fully normalized after `At::structurally_normalize` was called. + +Because this may result in behavior changes depending on how the trait solver handles +ambiguity, it is safer to also require full normalization there. This happens in +`FnCtxt::structurally_resolve_type` which always emits a hard error if the self type ends +up as an inference variable. There are some existing places which have a fallback for +inference variables instead. These places use `try_structurally_resolve_type` instead. + +## Why deep normalization with ambiguity is hard + +Fully correct deep normalization is very challenging, especially with the new solver +given that we do not want to deeply normalize inside of the solver. Mostly deeply normalizing +but sometimes failing to do so is bound to cause very hard to minimize and understand bugs. +If possible, avoiding any reliance on deep normalization entirely therefore feels preferable. + +If the solver itself does not deeply normalize, any inference constraints returned by the +solver would require normalization. Handling this correctly is ugly. This also means that +we change goals we provide to the trait solver by "normalizing away" some projections. + +The way we (mostly) guarantee deep normalization with the old solver is by eagerly replacing +the projection with an inference variable and emitting a nested `Projection` goal. This works +as `Projection` goals in the old solver deeply normalize. Unless we add another `PredicateKind` +for deep normalization to the new solver we cannot emulate this behavior. This does not work +for projections with bound variables, sometimes leaving them unnormalized. An approach which +also supports projections with bound variables will be even more involved. + + +[`assemble_candidates_after_normalizing_self_ty`]: https://github.com/rust-lang/rust/blob/1b6d4cdc4d923c148198ad4df230af48cdaca59e/compiler/rustc_trait_selection/src/solve/assembly/mod.rs#L330-L378 +[`AliasRelate`]: https://github.com/rust-lang/rust/blob/1b6d4cdc4d923c148198ad4df230af48cdaca59e/compiler/rustc_trait_selection/src/solve/alias_relate.rs#L16-L102 +[`normalizes-to` candidates]: https://github.com/rust-lang/rust/blob/1b6d4cdc4d923c148198ad4df230af48cdaca59e/compiler/rustc_trait_selection/src/solve/alias_relate.rs#L105-L151 +[the behavior of projection clauses]: https://github.com/rust-lang/trait-system-refactor-initiative/issues/1 +[normalize-via-infer]: https://github.com/rust-lang/rust/blob/1b6d4cdc4d923c148198ad4df230af48cdaca59e/compiler/rustc_trait_selection/src/solve/assembly/mod.rs#L350-L358 + +[^0]: TODO: currently refactoring this to use `NormalizesTo` predicates instead. \ No newline at end of file diff --git a/src/solve/proof-trees.md b/src/solve/proof-trees.md new file mode 100644 index 000000000..e0904d946 --- /dev/null +++ b/src/solve/proof-trees.md @@ -0,0 +1,50 @@ +# Proof trees + +The trait solver can optionally emit a "proof tree", a tree representation of what +happened while trying to prove a goal. + +The used datastructures for which are currently stored in +[`rustc_middle::traits::solve::inspect`]. + +## What are they used for + +There are 3 intended uses for proof trees. These uses are not yet implemented as +the representation of proof trees itself is currently still unstable. + +They should be used by type system diagnostics to get information about +why a goal failed or remained ambiguous. They should be used by rustdoc to get the +auto-trait implementations for user-defined types, and they should be usable to +vastly improve the debugging experience of the trait solver. + +For debugging you can use `-Zdump-solver-proof-tree` which dumps the proof tree +for all goals proven by the trait solver in the current session. + +## Requirements and design constraints for proof trees + +The trait solver uses [Canonicalization] and uses completely separate `InferCtxt` for +each nested goal. Both diagnostics and auto-traits in rustdoc need to correctly +handle "looking into nested goals". Given a goal like `Vec>: Debug`, we +canonicalize to `exists Vec>: Debug`, instantiate that goal as +`Vec>: Debug`, get a nested goal `Vec: Debug`, canonicalize this to get +`exists Vec: Debug`, instantiate this as `Vec: Debug` which then results +in a nested `?0: Debug` goal which is ambiguous. + +We need to be able to figure out that `?x` corresponds to `?0` in the nested queries. + +The debug output should also accurately represent the state at each point in the solver. +This means that even though a goal like `fn(?0): FnOnce(i32)` infers `?0` to `i32`, the +proof tree should still store `fn(): FnOnce(i32)` instead of +`fn(i32): FnOnce(i32)` until we actually infer `?0` to `i32`. + +## The current implementation and how to extract information from proof trees. + +Proof trees will be quite involved as they should accurately represent everything the +trait solver does, which includes fixpoint iterations and performance optimizations. + +We intend to provide a lossy user interface for all usecases. + +TODO: implement this user interface and explain how it can be used here. + + +[`rustc_middle::traits::solve::inspect`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/traits/solve/inspect/index.html +[Canonicalization]: ./canonicalization.md \ No newline at end of file diff --git a/src/solve/the-solver.md b/src/solve/the-solver.md new file mode 100644 index 000000000..f7d82d117 --- /dev/null +++ b/src/solve/the-solver.md @@ -0,0 +1,76 @@ +# The solver + +Also consider reading the documentation for [the recursive solver in chalk][chalk] +as it is very similar to this implementation and also talks about limitations of this +approach. + +[chalk]: https://rust-lang.github.io/chalk/book/recursive.html + +## A rough walkthrough + +The entry-point of the solver is `InferCtxtEvalExt::evaluate_root_goal`. This +function sets up the root `EvalCtxt` and then calls `EvalCtxt::evaluate_goal`, +to actually enter the trait solver. + +`EvalCtxt::evaluate_goal` handles [canonicalization](./canonicalization.md), caching, +overflow, and solver cycles. Once that is done, it creates a nested `EvalCtxt` with a +separate local `InferCtxt` and calls `EvalCtxt::compute_goal`, which is responsible for the +'actual solver behavior'. We match on the `PredicateKind`, delegating to a separate function +for each one. + +For trait goals, such a `Vec: Clone`, `EvalCtxt::compute_trait_goal` has +to collect all the possible ways this goal can be proven via +`EvalCtxt::assemble_and_evaluate_candidates`. Each candidate is handled in +a separate "probe", to not leak inference constraints to the other candidates. +We then try to merge the assembled candidates via `EvalCtxt::merge_candidates`. + + +## Important concepts and design pattern + +### `EvalCtxt::add_goal` + +To prove nested goals, we don't directly call `EvalCtxt::compute_goal`, but instead +add the goal to the `EvalCtxt` with `EvalCtxt::all_goal`. We then prove all nested +goals together in either `EvalCtxt::try_evaluate_added_goals` or +`EvalCtxt::evaluate_added_goals_and_make_canonical_response`. This allows us to handle +inference constraints from later goals. + +E.g. if we have both `?x: Debug` and `(): ConstrainToU8` as nested goals, +then proving `?x: Debug` is initially ambiguous, but after proving `(): ConstrainToU8` +we constrain `?x` to `u8` and proving `u8: Debug` succeeds. + +### Matching on `TyKind` + +We lazily normalize types in the solver, so we always have to assume that any types +and constants are potentially unnormalized. This means that matching on `TyKind` can easily +be incorrect. + +We handle normalization in two different ways. When proving `Trait` goals when normalizing +associated types, we separately assemble candidates depending on whether they structurally +match the self type. Candidates which match on the self type are handled in +`EvalCtxt::assemble_candidates_via_self_ty` which recurses via +`EvalCtxt::assemble_candidates_after_normalizing_self_ty`, which normalizes the self type +by one level. In all other cases we have to match on a `TyKind` we first use +`EvalCtxt::try_normalize_ty` to normalize the type as much as possible. + +### Higher ranked goals + +In case the goal is higher-ranked, e.g. `for<'a> F: FnOnce(&'a ())`, `EvalCtxt::compute_goal` +eagerly instantiates `'a` with a placeholder and then recursively proves +`F: FnOnce(&'!a ())` as a nested goal. + +### Dealing with choice + +Some goals can be proven in multiple ways. In these cases we try each option in +a separate "probe" and then attempt to merge the resulting responses by using +`EvalCtxt::try_merge_responses`. If merging the responses fails, we use +`EvalCtxt::flounder` instead, returning ambiguity. For some goals, we try +incompletely prefer some choices over others in case `EvalCtxt::try_merge_responses` +fails. + +## Learning more + +The solver should be fairly self-contained. I hope that the above information provides a +good foundation when looking at the code itself. Please reach out on zulip if you get stuck +while doing so or there are some quirks and design decisions which were unclear and deserve +better comments or should be mentioned here. diff --git a/src/solve/trait-solving.md b/src/solve/trait-solving.md new file mode 100644 index 000000000..7c1e0b684 --- /dev/null +++ b/src/solve/trait-solving.md @@ -0,0 +1,47 @@ +# Trait solving (new) + +This chapter describes how trait solving works with the new WIP solver located in +[`rustc_trait_selection/solve`][solve]. Feel free to also look at the docs for +[the current solver](../traits/resolution.md) and [the chalk solver](../traits/chalk.md) +can be found separately. + +## Core concepts + +The goal of the trait system is to check whether a given trait bound is satisfied. +Most notably when typechecking the body of - potentially generic - functions. +For example: + +```rust +fn uses_vec_clone(x: Vec) -> (Vec, Vec) { + (x.clone(), x) +} +``` +Here the call to `x.clone()` requires us to prove that `Vec` implements `Clone` given +the assumption that `T: Clone` is true. We can assume `T: Clone` as that will be proven by +callers of this function. + +The concept of "prove the `Vec: Clone` with the assumption `T: Clone`" is called a [`Goal`]. +Both `Vec: Clone` and `T: Clone` are represented using [`Predicate`]. There are other +predicates, most notably equality bounds on associated items: ` as IntoIterator>::Item == T`. +See the `PredicateKind` enum for an exhaustive list. A `Goal` is represented as the `predicate` we +have to prove and the `param_env` in which this predicate has to hold. + +We prove goals by checking whether each possible [`Candidate`] applies for the given goal by +recursively proving its nested goals. For a list of possible candidates with examples, look at +[`CandidateSource`]. The most important candidates are `Impl` candidates, i.e. trait implementations +written by the user, and `ParamEnv` candidates, i.e. assumptions in our current environment. + +Looking at the above example, to prove `Vec: Clone` we first use +`impl Clone for Vec`. To use this impl we have to prove the nested +goal that `T: Clone` holds. This can use the assumption `T: Clone` from the `ParamEnv` +which does not have any nested goals. Therefore `Vec: Clone` holds. + +The trait solver can either return success, ambiguity or an error as a [`CanonicalResponse`]. +For success and ambiguity it also returns constraints inference and region constraints. + +[solve]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/solve/index.html +[`Goal`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/solve/struct.Goal.html +[`Predicate`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Predicate.html +[`Candidate`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/solve/assembly/struct.Candidate.html +[`CandidateSource`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/solve/assembly/enum.CandidateSource.html +[`CanonicalResponse`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/solve/type.CanonicalResponse.html diff --git a/src/stability.md b/src/stability.md index 85c75fadb..bf82d8517 100644 --- a/src/stability.md +++ b/src/stability.md @@ -1,12 +1,12 @@ # Stability attributes - - This section is about the stability attributes and schemes that allow stable APIs to use unstable APIs internally in the rustc standard library. -For instructions on stabilizing a language feature see [Stabilizing -Features](./stabilization_guide.md). +**NOTE**: this section is for *library* features, not *language* features. For instructions on +stabilizing a language feature see [Stabilizing Features](./stabilization_guide.md). + + ## unstable diff --git a/src/stabilization_guide.md b/src/stabilization_guide.md index 0ac19293b..9bc70f65c 100644 --- a/src/stabilization_guide.md +++ b/src/stabilization_guide.md @@ -1,6 +1,6 @@ # Request for stabilization -**NOTE**: this page is about stabilizing language features. +**NOTE**: this page is about stabilizing *language* features. For stabilizing *library* features, see [Stabilizing a library feature]. [Stabilizing a library feature]: ./stability.md#stabilizing-a-library-feature @@ -109,7 +109,7 @@ to stabilize, something like (this example is taken from ```rust,ignore // pub(restricted) visibilities (RFC 1422) -(active, pub_restricted, "CURRENT_RUSTC_VERSION", Some(32409)), +(unstable, pub_restricted, "CURRENT_RUSTC_VERSION", Some(32409)), ``` The above line should be moved down to the area for "accepted" diff --git a/src/tests/adding.md b/src/tests/adding.md index 5bcc9b878..fc0c4937e 100644 --- a/src/tests/adding.md +++ b/src/tests/adding.md @@ -12,7 +12,7 @@ This will depend on the nature of the change and what you want to exercise. Here are some rough guidelines: - The majority of compiler tests are done with [compiletest]. - - The majority of compiletest tests are [UI](ui.md) tests in the [`src/test/ui`] directory. + - The majority of compiletest tests are [UI](ui.md) tests in the [`tests/ui`] directory. - Changes to the standard library are usually tested within the standard library itself. - The majority of standard library tests are written as doctests, which illustrate and exercise typical API behavior. @@ -30,7 +30,7 @@ Here are some rough guidelines: - Check out the [compiletest] chapter for more specialized test suites. [compiletest]: compiletest.md -[`src/test/ui`]: https://github.com/rust-lang/rust/tree/master/src/test/ui/ +[`tests/ui`]: https://github.com/rust-lang/rust/tree/master/tests/ui/ ## UI test walkthrough @@ -41,13 +41,13 @@ For this tutorial, we'll be adding a test for an async error message. ### Step 1. Add a test file The first step is to create a Rust source file somewhere in the -[`src/test/ui`] tree. +[`tests/ui`] tree. When creating a test, do your best to find a good location and name (see [Test organization](ui.md#test-organization) for more). Since naming is the hardest part of development, everything should be downhill from here! -Let's place our async test at `src/test/ui/async-await/await-without-async.rs`: +Let's place our async test at `tests/ui/async-await/await-without-async.rs`: ```rust,ignore // Check what happens when using await in a non-async fn. @@ -84,17 +84,17 @@ The next step is to create the expected output from the compiler. This can be done with the `--bless` option: ```sh -./x.py test src/test/ui/async-await/await-without-async.rs --bless +./x test tests/ui/async-await/await-without-async.rs --bless ``` This will build the compiler (if it hasn't already been built), compile the test, and place the output of the compiler in a file called -`src/test/ui/async-await/await-without-async.stderr`. +`tests/ui/async-await/await-without-async.stderr`. However, this step will fail! You should see an error message, something like this: -> error: /rust/src/test/ui/async-await/await-without-async.rs:7: unexpected +> error: /rust/tests/ui/async-await/await-without-async.rs:7: unexpected > error: '7:10: 7:16: `await` is only allowed inside `async` functions and > blocks E0728' @@ -118,7 +118,7 @@ annotations](ui.md#error-annotations) section). Save that, and run the test again: ```sh -./x.py test src/test/ui/async-await/await-without-async.rs +./x test tests/ui/async-await/await-without-async.rs ``` It should now pass, yay! @@ -131,7 +131,7 @@ If you are adding a new diagnostic message, now would be a good time to also consider how readable the message looks overall, particularly for people new to Rust. -Our example `src/test/ui/async-await/await-without-async.stderr` file should +Our example `tests/ui/async-await/await-without-async.stderr` file should look like this: ```text @@ -166,7 +166,7 @@ The final step before posting a PR is to check if you have affected anything els Running the UI suite is usually a good start: ```sh -./x.py test src/test/ui +./x test tests/ui ``` If other tests start failing, you may need to investigate what has changed diff --git a/src/tests/ci.md b/src/tests/ci.md index cca4973cb..bfdc2a45a 100644 --- a/src/tests/ci.md +++ b/src/tests/ci.md @@ -6,7 +6,7 @@ When a Pull Request is opened on GitHub, [GitHub Actions] will automatically launch a build that will run all tests on some configurations (x86_64-gnu-llvm-13 linux, x86_64-gnu-tools linux, and mingw-check linux). -In essence, each runs `./x.py test` with various different options. +In essence, each runs `./x test` with various different options. The integration bot [bors] is used for coordinating merges to the master branch. When a PR is approved, it goes into a [queue] where merges are tested one at a @@ -54,8 +54,8 @@ the other jobs. The comment at the top of `ci.yml` will tell you to run this command: ```sh -./x.py run src/tools/expand-yaml-anchors -```` +./x run src/tools/expand-yaml-anchors +``` This will generate the true [`.github/workflows/ci.yml`] which is what GitHub Actions uses. diff --git a/src/tests/compiletest.md b/src/tests/compiletest.md index 70cef2ad3..7b6ed435d 100644 --- a/src/tests/compiletest.md +++ b/src/tests/compiletest.md @@ -11,6 +11,16 @@ efficient test execution (parallel execution is supported), and allows the test author to configure behavior and expected results of both individual and groups of tests. +> NOTE: +> For macOS users, `SIP` (System Integrity Protection) [may consistently +> check the compiled binary by sending network requests to Apple][zulip], +> so you may get a huge performance degradation when running tests. +> +> You can resolve it by tweaking the following settings: +> `Privacy & Security -> Developer Tools -> Add Terminal (Or VsCode, etc.)`. + +[zulip]: https://rust-lang.zulipchat.com/#narrow/stream/182449-t-compiler.2Fhelp/topic/.E2.9C.94.20Is.20there.20any.20performance.20issue.20for.20MacOS.3F + `compiletest` may check test code for success, for runtime failure, or for compile-time failure. Tests are typically organized as a Rust source file with annotations in @@ -24,13 +34,18 @@ See the [Adding new tests](adding.md) chapter for a tutorial on creating a new test, and the [Running tests](running.md) chapter on how to run the test suite. +Compiletest itself tries to avoid running tests when the artifacts +that are involved (mainly the compiler) haven't changed. You can use +`x test --test-args --force-rerun` to rerun a test even when none of the +inputs have changed. + ## Test suites -All of the tests are in the [`src/test`] directory. +All of the tests are in the [`tests`] directory. The tests are organized into "suites", with each suite in a separate subdirectory. Each test suite behaves a little differently, with different compiler behavior and different checks for correctness. -For example, the [`src/test/incremental`] directory contains tests for +For example, the [`tests/incremental`] directory contains tests for incremental compilation. The various suites are defined in [`src/tools/compiletest/src/common.rs`] in the `pub enum Mode` declaration. @@ -52,6 +67,9 @@ The following test suites are available, with links for more information: - `run-make-fulldeps` — `run-make` tests which require a linkable build of `rustc`, or the rust demangler - [`run-pass-valgrind`](#valgrind-tests) — tests run with Valgrind +- [`coverage`](#coverage-tests) - tests for coverage instrumentation +- [`coverage-run-rustdoc`](#coverage-tests) - coverage tests that also run + instrumented doctests - [Rustdoc tests](../rustdoc.md#tests): - `rustdoc` — tests for rustdoc, making sure that the generated files contain the expected documentation. @@ -62,12 +80,12 @@ The following test suites are available, with links for more information: - `rustdoc-json` — tests on the JSON output of rustdoc. - `rustdoc-ui` — tests on the terminal output of rustdoc. -[`src/test`]: https://github.com/rust-lang/rust/blob/master/src/test +[`tests`]: https://github.com/rust-lang/rust/blob/master/tests [`src/tools/compiletest/src/common.rs`]: https://github.com/rust-lang/rust/tree/master/src/tools/compiletest/src/common.rs ### Pretty-printer tests -The tests in [`src/test/pretty`] exercise the "pretty-printing" functionality of `rustc`. +The tests in [`tests/pretty`] exercise the "pretty-printing" functionality of `rustc`. The `-Z unpretty` CLI option for `rustc` causes it to translate the input source into various different formats, such as the Rust source after macro expansion. @@ -119,11 +137,11 @@ The header commands for pretty-printing tests are: pretty-printing rounds will be compared to ensure that the pretty-printed output converges to a steady state. -[`src/test/pretty`]: https://github.com/rust-lang/rust/tree/master/src/test/pretty +[`tests/pretty`]: https://github.com/rust-lang/rust/tree/master/tests/pretty ### Incremental tests -The tests in [`src/test/incremental`] exercise incremental compilation. +The tests in [`tests/incremental`] exercise incremental compilation. They use [revision headers](#revisions) to tell compiletest to run the compiler in a series of steps. Compiletest starts with an empty directory with the `-C incremental` flag, and @@ -168,12 +186,12 @@ cause an Internal Compiler Error (ICE). This is a highly specialized header to check that the incremental cache continues to work after an ICE. -[`src/test/incremental`]: https://github.com/rust-lang/rust/tree/master/src/test/incremental +[`tests/incremental`]: https://github.com/rust-lang/rust/tree/master/tests/incremental ### Debuginfo tests -The tests in [`src/test/debuginfo`] test debuginfo generation. +The tests in [`tests/debuginfo`] test debuginfo generation. They build a program, launch a debugger, and issue commands to the debugger. A single test can work with cdb, gdb, and lldb. @@ -237,12 +255,12 @@ test based on the debugger currently being used: NOTE: The "Rust" version of LLDB doesn't exist anymore, so this will always be ignored. This should probably be removed. -[`src/test/debuginfo`]: https://github.com/rust-lang/rust/tree/master/src/test/debuginfo +[`tests/debuginfo`]: https://github.com/rust-lang/rust/tree/master/tests/debuginfo ### Codegen tests -The tests in [`src/test/codegen`] test LLVM code generation. +The tests in [`tests/codegen`] test LLVM code generation. They compile the test with the `--emit=llvm-ir` flag to emit LLVM IR. They then run the LLVM [FileCheck] tool. The test is annotated with various `// CHECK` comments to check the generated code. @@ -250,13 +268,13 @@ See the FileCheck documentation for a tutorial and more information. See also the [assembly tests](#assembly-tests) for a similar set of tests. -[`src/test/codegen`]: https://github.com/rust-lang/rust/tree/master/src/test/codegen +[`tests/codegen`]: https://github.com/rust-lang/rust/tree/master/tests/codegen [FileCheck]: https://llvm.org/docs/CommandGuide/FileCheck.html ### Assembly tests -The tests in [`src/test/assembly`] test LLVM assembly output. +The tests in [`tests/assembly`] test LLVM assembly output. They compile the test with the `--emit=asm` flag to emit a `.s` file with the assembly output. They then run the LLVM [FileCheck] tool. @@ -271,12 +289,12 @@ See the FileCheck documentation for a tutorial and more information. See also the [codegen tests](#codegen-tests) for a similar set of tests. -[`src/test/assembly`]: https://github.com/rust-lang/rust/tree/master/src/test/assembly +[`tests/assembly`]: https://github.com/rust-lang/rust/tree/master/tests/assembly ### Codegen-units tests -The tests in [`src/test/codegen-units`] test the +The tests in [`tests/codegen-units`] test the [monomorphization](../backend/monomorph.md) collector and CGU partitioning. These tests work by running `rustc` with a flag to print the result of the @@ -295,12 +313,12 @@ where `cgu` is a space separated list of the CGU names and the linkage information in brackets. For example: `//~ MONO_ITEM static function::FOO @@ statics[Internal]` -[`src/test/codegen-units`]: https://github.com/rust-lang/rust/tree/master/src/test/codegen-units +[`tests/codegen-units`]: https://github.com/rust-lang/rust/tree/master/tests/codegen-units ### Mir-opt tests -The tests in [`src/test/mir-opt`] check parts of the generated MIR to make +The tests in [`tests/mir-opt`] check parts of the generated MIR to make sure it is generated correctly and is doing the expected optimizations. Check out the [MIR Optimizations](../mir/optimizations.md) chapter for more. @@ -315,7 +333,7 @@ set a baseline for optimizations: The test should be annotated with `// EMIT_MIR` comments that specify files that will contain the expected MIR output. -You can use `x.py test --bless` to create the initial expected files. +You can use `x test --bless` to create the initial expected files. There are several forms the `EMIT_MIR` comment can take: @@ -345,19 +363,19 @@ problematic in the presence of pointers in constants or other bit width dependent things. In that case you can add `// EMIT_MIR_FOR_EACH_BIT_WIDTH` to your test, causing separate files to be generated for 32bit and 64bit systems. -[`src/test/mir-opt`]: https://github.com/rust-lang/rust/tree/master/src/test/mir-opt +[`tests/mir-opt`]: https://github.com/rust-lang/rust/tree/master/tests/mir-opt -### Run-make tests +### `run-make` tests -The tests in [`src/test/run-make`] are general-purpose tests using Makefiles +The tests in [`tests/run-make`] are general-purpose tests using Makefiles which provide the ultimate in flexibility. These should be used as a last resort. If possible, you should use one of the other test suites. If there is some minor feature missing which you need for your test, consider extending compiletest to add a header command for what you need. -However, sometimes just running a bunch of commands is really what you -need, `run-make` is here to the rescue! +However, if running a bunch of commands is really what you need, +`run-make` is here to the rescue! Each test should be in a separate directory with a `Makefile` indicating the commands to run. @@ -365,18 +383,89 @@ There is a [`tools.mk`] Makefile which you can include which provides a bunch of utilities to make it easier to run commands and compare outputs. Take a look at some of the other tests for some examples on how to get started. -[`tools.mk`]: https://github.com/rust-lang/rust/blob/master/src/test/run-make-fulldeps/tools.mk -[`src/test/run-make`]: https://github.com/rust-lang/rust/tree/master/src/test/run-make +[`tools.mk`]: https://github.com/rust-lang/rust/blob/master/tests/run-make/tools.mk +[`tests/run-make`]: https://github.com/rust-lang/rust/tree/master/tests/run-make ### Valgrind tests -The tests in [`src/test/run-pass-valgrind`] are for use with [Valgrind]. +The tests in [`tests/run-pass-valgrind`] are for use with [Valgrind]. These are currently vestigial, as Valgrind is no longer used in CI. These may be removed in the future. [Valgrind]: https://valgrind.org/ -[`src/test/run-pass-valgrind`]: https://github.com/rust-lang/rust/tree/master/src/test/run-pass-valgrind +[`tests/run-pass-valgrind`]: https://github.com/rust-lang/rust/tree/master/tests/run-pass-valgrind + + +### Coverage tests + +The tests in [`tests/coverage`] are shared by multiple test modes that test +coverage instrumentation in different ways. +Running the `coverage` test suite will automatically run each test in all of +the different coverage modes. + +Each mode also has an alias to run the coverage tests in just that mode: + +```bash +./x test coverage # runs all of tests/coverage in all coverage modes +./x test tests/coverage # same as above + +./x test tests/coverage/if.rs # runs the specified test in all coverage modes + +./x test coverage-map # runs all of tests/coverage in "coverage-map" mode only +./x test coverage-run # runs all of tests/coverage in "coverage-run" mode only + +./x test coverage-map -- tests/coverage/if.rs # runs the specified test in "coverage-map" mode only +``` + +--- + +In `coverage-map` mode, these tests verify the mappings between source code +regions and coverage counters that are emitted by LLVM. +They compile the test with `--emit=llvm-ir`, +then use a custom tool ([`src/tools/coverage-dump`]) +to extract and pretty-print the coverage mappings embedded in the IR. +These tests don't require the profiler runtime, so they run in PR CI jobs and +are easy to run/bless locally. + +These coverage map tests can be sensitive to changes in MIR lowering or MIR +optimizations, producing mappings that are different but produce identical +coverage reports. + +As a rule of thumb, any PR that doesn't change coverage-specific +code should **feel free to re-bless** the `coverage-map` tests as necessary, +without worrying about the actual changes, as long as the `coverage-run` tests +still pass. + +--- + +In `coverage-run` mode, these tests perform an end-to-end test of coverage reporting. +They compile a test program with coverage instrumentation, run that program to +produce raw coverage data, and then use LLVM tools to process that data into a +human-readable code coverage report. + +Instrumented binaries need to be linked against the LLVM profiler runtime, +so `coverage-run` tests are **automatically skipped** +unless the profiler runtime is enabled in `config.toml`: + +```toml +# config.toml +[build] +profiler = true +``` + +This also means that they typically don't run in PR CI jobs, +though they do run as part of the full set of CI jobs used for merging. + +--- + +The tests in [`tests/coverage-run-rustdoc`] also run instrumented doctests and +include them in the coverage report. This avoids having to build rustdoc when +only running the main `coverage` suite. + +[`tests/coverage`]: https://github.com/rust-lang/rust/tree/master/tests/coverage +[`src/tools/coverage-dump`]: https://github.com/rust-lang/rust/tree/master/src/tools/coverage-dump +[`tests/coverage-run-rustdoc`]: https://github.com/rust-lang/rust/tree/master/tests/coverage-run-rustdoc ## Building auxiliary crates @@ -437,7 +526,7 @@ compiler, and `no-prefer-dynamic` is needed to tell compiletest to not use `prefer-dynamic` which is not compatible with proc-macros. The `#![crate_type]` attribute is needed to specify the correct crate-type. -Then in your test, you can build with with `aux-build`: +Then in your test, you can build with `aux-build`: ```rust,ignore // aux-build: my-proc-macro.rs @@ -452,9 +541,6 @@ fn main() { ## Revisions -Certain classes of tests support "revisions" (as of July 2022, -this includes UI, assembly, codegen, debuginfo, incremental, and rustdoc UI tests, -though incremental tests are somewhat different). Revisions allow a single test file to be used for multiple tests. This is done by adding a special header at the top of the file: @@ -488,10 +574,18 @@ currently only apply to the test as a whole, not to particular revisions. The only headers that are intended to really work when customized to a revision are error patterns and compiler flags. + +Following is classes of tests that support revisions: +- UI +- assembly +- codegen +- debuginfo +- rustdoc UI tests +- incremental (these are special in that they inherently cannot be run in parallel) ## Compare modes -Compiletest can be run in different modes, called *compare modes*, which can +Compiletest can be run in different modes, called _compare modes_, which can be used to compare the behavior of all tests with different compiler flags enabled. This can help highlight what differences might appear with certain flags, and @@ -501,7 +595,7 @@ To run the tests in a different mode, you need to pass the `--compare-mode` CLI flag: ```bash -./x.py test src/test/ui --compare-mode=chalk +./x test tests/ui --compare-mode=chalk ``` The possible compare modes are: @@ -517,10 +611,10 @@ tests support different output for different modes. In CI, compare modes are only used in one Linux builder, and only with the following settings: -* `src/test/debuginfo`: Uses `split-dwarf` mode. +* `tests/debuginfo`: Uses `split-dwarf` mode. This helps ensure that none of the debuginfo tests are affected when enabling split-DWARF. Note that compare modes are separate to [revisions](#revisions). -All revisions are tested when running `./x.py test src/test/ui`, however +All revisions are tested when running `./x test tests/ui`, however compare-modes must be manually run individually via the `--compare-mode` flag. diff --git a/src/tests/docker.md b/src/tests/docker.md index feb614c51..31e3825f5 100644 --- a/src/tests/docker.md +++ b/src/tests/docker.md @@ -21,7 +21,7 @@ directory. From there, you can run `../src/ci/run.sh` which will run the build as defined by the image. Alternatively, you can run individual commands to do specific tasks. For -example, you can run `python3 ../x.py test src/test/ui` to just run UI tests. +example, you can run `../x test tests/ui` to just run UI tests. Note that there is some configuration in the [`src/ci/run.sh`] script that you may need to recreate. Particularly, set `submodules = false` in your `config.toml` so that it doesn't attempt to modify the read-only directory. diff --git a/src/tests/headers.md b/src/tests/headers.md index bfa637f34..fce2397e5 100644 --- a/src/tests/headers.md +++ b/src/tests/headers.md @@ -28,7 +28,7 @@ Header commands can be standalone (like `// run-pass`) or take a value (like ## Header commands The following is a list of header commands. -Commands are linked to sections the describe the command in more detail if available. +Commands are linked to sections that describe the command in more detail if available. This list may not be exhaustive. Header commands can generally be found by browsing the `TestProps` structure found in [`header.rs`] from the compiletest source. @@ -52,6 +52,8 @@ found in [`header.rs`] from the compiletest source. * [`stderr-per-bitwidth`](ui.md#output-comparison) — separate output per bit width * [`dont-check-compiler-stderr`](ui.md#output-comparison) — don't validate stderr * [`dont-check-compiler-stdout`](ui.md#output-comparison) — don't validate stdout + * [`compare-output-lines-by-subset`](ui.md#output-comparison) — checks output by + line subset * [Building auxiliary crates](compiletest.md#building-auxiliary-crates) * `aux-build` * `aux-crate` @@ -68,10 +70,10 @@ found in [`header.rs`] from the compiletest source. * `min-llvm-versionX` * `min-system-llvm-version` * `ignore-llvm-version` - * `ignore-llvm-version` * [Environment variable headers](#environment-variable-headers) * `rustc-env` * `exec-env` + * `unset-exec-env` * `unset-rustc-env` * [Miscellaneous headers](#miscellaneous-headers) * `compile-flags` — adds compiler flags @@ -142,7 +144,7 @@ The following header commands will check rustc build settings and target setting (AddressSanitizer, hardware-assisted AddressSanitizer, LeakSanitizer, MemorySanitizer or ThreadSanitizer respectively) * `needs-run-enabled` — ignores if it is a test that gets executed, and - running has been disabled. Running tests can be disabled with the `x.py test + running has been disabled. Running tests can be disabled with the `x test --run=never` flag, or running on fuchsia. * `needs-unwind` — ignores if the target does not support unwinding * `needs-rust-lld` — ignores if the rust lld support is not enabled @@ -177,6 +179,8 @@ The following headers affect environment variables. form `KEY=VALUE`. * `exec-env` is an environment variable to set when executing a test of the form `KEY=VALUE`. +* `unset-exec-env` specifies an environment variable to unset when executing a + test. * `unset-rustc-env` specifies an environment variable to unset when running `rustc`. @@ -186,7 +190,7 @@ The following headers are generally available, and not specific to particular test suites. * `compile-flags` passes extra command-line args to the compiler, - e.g. `compile-flags -g` which forces debuginfo to be enabled. + e.g. `// compile-flags: -g` which forces debuginfo to be enabled. * `run-flags` passes extra args to the test if the test is to be executed. * `edition` controls the edition the test should be compiled with (defaults to 2015). Example usage: `// edition:2018`. @@ -244,12 +248,12 @@ described below: - Examples: `/path/to/rust`, `/path/to/build/root` - `{{src-base}}`: The directory where the test is defined. This is equivalent to `$DIR` for [output normalization]. - - Example: `/path/to/rust/src/test/ui/error-codes` + - Example: `/path/to/rust/tests/ui/error-codes` - `{{build-base}}`: The base directory where the test's output goes. This is equivalent to `$TEST_BUILD_DIR` for [output normalization]. - Example: `/path/to/rust/build/x86_64-unknown-linux-gnu/test/ui` -See [`src/test/ui/commandline-argfile.rs`](https://github.com/rust-lang/rust/blob/a5029ac0ab372aec515db2e718da6d7787f3d122/src/test/ui/commandline-argfile.rs) +See [`tests/ui/commandline-argfile.rs`](https://github.com/rust-lang/rust/blob/master/tests/ui/commandline-argfile.rs) for an example of a test that uses this substitution. [output normalization]: ui.md#normalization diff --git a/src/tests/intro.md b/src/tests/intro.md index 66e0a9eef..647d1fbea 100644 --- a/src/tests/intro.md +++ b/src/tests/intro.md @@ -3,49 +3,49 @@ The Rust project runs a wide variety of different tests, orchestrated by -the build system (`./x.py test`). +the build system (`./x test`). This section gives a brief overview of the different testing tools. Subsequent chapters dive into [running tests](running.md) and [adding new tests](adding.md). ## Kinds of tests There are several kinds of tests to exercise things in the Rust distribution. -Almost all of them are driven by `./x.py test`, with some exceptions noted below. +Almost all of them are driven by `./x test`, with some exceptions noted below. ### Compiletest The main test harness for testing the compiler itself is a tool called [compiletest]. It supports running different styles of tests, called *test suites*. -The tests are all located in the [`src/test`] directory. +The tests are all located in the [`tests`] directory. The [Compiletest chapter][compiletest] goes into detail on how to use this tool. -> Example: `./x.py test src/test/ui` +> Example: `./x test tests/ui` [compiletest]: compiletest.md -[`src/test`]: https://github.com/rust-lang/rust/tree/master/src/test +[`tests`]: https://github.com/rust-lang/rust/tree/master/tests ### Package tests The standard library and many of the compiler packages include typical Rust `#[test]` unit tests, integration tests, and documentation tests. You can pass a path to `x.py` to almost any package in the `library` or `compiler` directory, -and `x.py` will essentially run `cargo test` on that package. +and `x` will essentially run `cargo test` on that package. Examples: -| Command | Description | -|---------|-------------| -| `./x.py test library/std` | Runs tests on `std` only | -| `./x.py test library/core` | Runs tests on `core` only | -| `./x.py test compiler/rustc_data_structures` | Runs tests on `rustc_data_structures` | +| Command | Description | +| ----------------------------------------- | ------------------------------------- | +| `./x test library/std` | Runs tests on `std` only | +| `./x test library/core` | Runs tests on `core` only | +| `./x test compiler/rustc_data_structures` | Runs tests on `rustc_data_structures` | The standard library relies very heavily on documentation tests to cover its functionality. However, unit tests and integration tests can also be used as needed. Almost all of the compiler packages have doctests disabled. -The standard library and compiler always place all unit tests in a separate `tests` file -(this is enforced in [tidy][tidy-unit-tests]). -This approach ensures that when the test file is changed, the crate does not need to be recompiled. +All standard library and compiler unit tests are placed in separate `tests` file +(which is enforced in [tidy][tidy-unit-tests]). +This ensures that when the test file is changed, the crate does not need to be recompiled. For example: ```rust,ignore @@ -53,12 +53,11 @@ For example: mod tests; ``` -If it wasn't done this way, and the tests were placed in the same file as the source, -then changing or adding a test would cause the crate you are working on to be recompiled. -If you were working on something like `core`, -then that would require recompiling the entire standard library, and the entirety of `rustc`. +If it wasn't done this way, +and you were working on something like `core`, +that would require recompiling the entire standard library, and the entirety of `rustc`. -`./x.py test` includes some CLI options for controlling the behavior with these tests: +`./x test` includes some CLI options for controlling the behavior with these tests: * `--doc` — Only runs documentation tests in the package. * `--no-doc` — Run all tests *except* documentation tests. @@ -71,7 +70,7 @@ Tidy is a custom tool used for validating source code style and formatting conve such as rejecting long lines. There is more information in the [section on coding conventions](../conventions.md#formatting). -> Example: `./x.py test tidy` +> Example: `./x test tidy` ### Formatting @@ -80,28 +79,28 @@ The formatting check is automatically run by the Tidy tool mentioned above. Examples: -| Command | Description | -|---------|-------------| -| `./x.py fmt --check` | Checks formatting and exits with an error if formatting is needed. | -| `./x.py fmt` | Runs rustfmt across the entire codebase. | -| `./x.py test tidy --bless` | First runs rustfmt to format the codebase, then runs tidy checks. | +| Command | Description | +| ----------------------- | ------------------------------------------------------------------ | +| `./x fmt --check` | Checks formatting and exits with an error if formatting is needed. | +| `./x fmt` | Runs rustfmt across the entire codebase. | +| `./x test tidy --bless` | First runs rustfmt to format the codebase, then runs tidy checks. | ### Book documentation tests All of the books that are published have their own tests, primarily for validating that the Rust code examples pass. Under the hood, these are essentially using `rustdoc --test` on the markdown files. -The tests can be run by passing a path to a book to `./x.py test`. +The tests can be run by passing a path to a book to `./x test`. -> Example: `./x.py test src/doc/book` +> Example: `./x test src/doc/book` ### Documentation link checker Links across all documentation is validated with a link checker tool. -> Example: `./x.py test src/tools/linkchecker` +> Example: `./x test src/tools/linkchecker` -> Example: `./x.py test linkchecker` +> Example: `./x test linkchecker` This requires building all of the documentation, which might take a while. @@ -110,7 +109,7 @@ This requires building all of the documentation, which might take a while. `distcheck` verifies that the source distribution tarball created by the build system will unpack, build, and run all tests. -> Example: `./x.py test distcheck` +> Example: `./x test distcheck` ### Tool tests @@ -119,9 +118,9 @@ This includes things such as cargo, clippy, rustfmt, miri, bootstrap (testing the Rust build system itself), etc. Most of the tools are located in the [`src/tools`] directory. -To run the tool's tests, just pass its path to `./x.py test`. +To run the tool's tests, just pass its path to `./x test`. -> Example: `./x.py test src/tools/cargo` +> Example: `./x test src/tools/cargo` Usually these tools involve running `cargo test` within the tool's directory. @@ -139,7 +138,7 @@ More information can be found in the [toolstate documentation]. (such as `servo`, `ripgrep`, `tokei`, etc.). This ensures there aren't any significant regressions. -> Example: `./x.py test src/tools/cargotest` +> Example: `./x test src/tools/cargotest` ### Crater diff --git a/src/tests/running.md b/src/tests/running.md index 5d1441936..71fb82b68 100644 --- a/src/tests/running.md +++ b/src/tests/running.md @@ -2,11 +2,11 @@ -You can run the tests using `x.py`. The most basic command – which +You can run the tests using `x`. The most basic command – which you will almost never want to use! – is as follows: ```bash -./x.py test +./x test ``` This will build the stage 1 compiler and then run the whole test @@ -37,7 +37,7 @@ modifying rustc to see if things are generally working correctly would be the following: ```bash -./x.py test src/test/ui +./x test tests/ui ``` This will run the `ui` test suite. Of course, the choice @@ -46,32 +46,32 @@ doing. For example, if you are hacking on debuginfo, you may be better off with the debuginfo test suite: ```bash -./x.py test src/test/debuginfo +./x test tests/debuginfo ``` If you only need to test a specific subdirectory of tests for any -given test suite, you can pass that directory to `./x.py test`: +given test suite, you can pass that directory to `./x test`: ```bash -./x.py test src/test/ui/const-generics +./x test tests/ui/const-generics ``` Likewise, you can test a single file by passing its path: ```bash -./x.py test src/test/ui/const-generics/const-test.rs +./x test tests/ui/const-generics/const-test.rs ``` ### Run only the tidy script ```bash -./x.py test tidy +./x test tidy ``` ### Run tests on the standard library ```bash -./x.py test --stage 0 library/std +./x test --stage 0 library/std ``` Note that this only runs tests on `std`; if you want to test `core` or other crates, @@ -80,13 +80,13 @@ you have to specify those explicitly. ### Run the tidy script and tests on the standard library ```bash -./x.py test --stage 0 tidy library/std +./x test --stage 0 tidy library/std ``` ### Run tests on the standard library using a stage 1 compiler ```bash -./x.py test --stage 1 library/std +./x test --stage 1 library/std ``` By listing which test suites you want to run you avoid having to run @@ -99,7 +99,7 @@ there are some limitations. ### Run all tests using a stage 2 compiler ```bash -./x.py test --stage 2 +./x test --stage 2 ``` You almost never need to do this; CI will run these tests for you. @@ -108,13 +108,13 @@ You almost never need to do this; CI will run these tests for you. You may want to run unit tests on a specific file with following: ```bash -./x.py test compiler/rustc_data_structures/src/thin_vec/tests.rs +./x test compiler/rustc_data_structures/src/thin_vec/tests.rs ``` But unfortunately, it's impossible. You should invoke following instead: ```bash -./x.py test compiler/rustc_data_structures/ --test-args thin_vec +./x test compiler/rustc_data_structures/ --test-args thin_vec ``` ## Running an individual test @@ -122,10 +122,10 @@ But unfortunately, it's impossible. You should invoke following instead: Another common thing that people want to do is to run an **individual test**, often the test they are trying to fix. As mentioned earlier, you may pass the full file path to achieve this, or alternatively one -may invoke `x.py` with the `--test-args` option: +may invoke `x` with the `--test-args` option: ```bash -./x.py test src/test/ui --test-args issue-1234 +./x test tests/ui --test-args issue-1234 ``` Under the hood, the test runner invokes the standard Rust test runner @@ -133,14 +133,23 @@ Under the hood, the test runner invokes the standard Rust test runner filtering for tests that include "issue-1234" in the name. (Thus `--test-args` is a good way to run a collection of related tests.) +## Passing arguments to `rustc` when running tests + +It can sometimes be useful to run some tests with specific compiler arguments, +without using `RUSTFLAGS` (during development of unstable features, with `-Z` +flags, for example). + +This can be done with `./x test`'s `--rustc-args` option, to pass additional +arguments to the compiler when building the tests. + ## Editing and updating the reference files If you have changed the compiler's output intentionally, or you are making a new test, you can pass `--bless` to the test subcommand. E.g. -if some tests in `src/test/ui` are failing, you can run +if some tests in `tests/ui` are failing, you can run ```text -./x.py test src/test/ui --bless +./x test tests/ui --bless ``` to automatically adjust the `.stderr`, `.stdout` or `.fixed` files of @@ -165,37 +174,16 @@ Pass UI tests now have three modes, `check-pass`, `build-pass` and `run-pass`. When `--pass $mode` is passed, these tests will be forced to run under the given `$mode` unless the directive `// ignore-pass` exists in the test file. For example, you can run all the tests in -`src/test/ui` as `check-pass`: +`tests/ui` as `check-pass`: ```bash -./x.py test src/test/ui --pass check +./x test tests/ui --pass check ``` By passing `--pass $mode`, you can reduce the testing time. For each mode, please see [Controlling pass/fail expectations](ui.md#controlling-passfail-expectations). -## Using incremental compilation - -You can further enable the `--incremental` flag to save additional -time in subsequent rebuilds: - -```bash -./x.py test src/test/ui --incremental --test-args issue-1234 -``` - -If you don't want to include the flag with every command, you can -enable it in the `config.toml`: - -```toml -[rust] -incremental = true -``` - -Note that incremental compilation will use more disk space than usual. -If disk space is a concern for you, you might want to check the size -of the `build` directory from time to time. - ## Running tests with different "compare modes" UI tests may have different output depending on certain "modes" that @@ -205,7 +193,7 @@ mode, a test `foo.rs` will first look for expected output in The following will run the UI test suite in Polonius mode: ```bash -./x.py test src/test/ui --compare-mode=polonius +./x test tests/ui --compare-mode=polonius ``` See [Compare modes](compiletest.md#compare-modes) for more details. @@ -218,7 +206,7 @@ Most tests are just `rs` files, so after you can do something like: ```bash -rustc +stage1 src/test/ui/issue-1234.rs +rustc +stage1 tests/ui/issue-1234.rs ``` This is much faster, but doesn't always work. For example, some tests @@ -238,11 +226,11 @@ execution* so be careful where it is used. To do this, first build `remote-test-server` for the remote machine, e.g. for RISC-V ```sh -./x.py build src/tools/remote-test-server --target riscv64gc-unknown-linux-gnu +./x build src/tools/remote-test-server --target riscv64gc-unknown-linux-gnu ``` The binary will be created at -`./build/$HOST_ARCH/stage2-tools/$TARGET_ARCH/release/remote-test-server`. Copy +`./build/host/stage2-tools/$TARGET_ARCH/release/remote-test-server`. Copy this over to the remote machine. On the remote machine, run the `remote-test-server` with the `--bind @@ -268,11 +256,11 @@ pong ``` To run tests using the remote runner, set the `TEST_DEVICE_ADDR` environment -variable then use `x.py` as usual. For example, to run `ui` tests for a RISC-V +variable then use `x` as usual. For example, to run `ui` tests for a RISC-V machine with the IP address `1.2.3.4` use ```sh export TEST_DEVICE_ADDR="1.2.3.4:12345" -./x.py test src/test/ui --target riscv64gc-unknown-linux-gnu +./x test tests/ui --target riscv64gc-unknown-linux-gnu ``` If `remote-test-server` was run with the verbose flag, output on the test machine @@ -294,7 +282,7 @@ run "/tmp/work/test1018/a" [...] ``` -Tests are built on the machine running `x.py` not on the remote machine. Tests +Tests are built on the machine running `x` not on the remote machine. Tests which fail to build unexpectedly (or `ui` tests producing incorrect build output) may fail without ever running on the remote machine. diff --git a/src/tests/suggest-tests.md b/src/tests/suggest-tests.md new file mode 100644 index 000000000..f6b6d1537 --- /dev/null +++ b/src/tests/suggest-tests.md @@ -0,0 +1,55 @@ +# Suggest tests tool + +This chapter is about the internals of and contribution instructions for the +`suggest-tests` tool. For a high-level overview of the tool, see +[this section](../building/suggested.md#x-suggest). This tool is currently in a +beta state and is tracked by [this](https://github.com/rust-lang/rust/issues/109933) +issue on Github. Currently the number of tests it will suggest are very limited +in scope, we are looking to expand this (contributions welcome!). + +## Internals + +The tool is defined in a separate crate ([`src/tools/suggest-tests`](https://github.com/rust-lang/rust/blob/master/src/tools/suggest-tests)) +which outputs suggestions which are parsed by a shim in bootstrap +([`src/bootstrap/suggest.rs`](https://github.com/rust-lang/rust/blob/master/src/bootstrap/suggest.rs)). +The only notable thing the bootstrap shim does is (when invoked with the +`--run` flag) use bootstrap's internal mechanisms to create a new `Builder` and +uses it to invoke the suggested commands. The `suggest-tests` crate is where the +fun happens, two kinds of suggestions are defined: "static" and "dynamic" +suggestions. + +### Static suggestions + +Defined [here](https://github.com/rust-lang/rust/blob/master/src/tools/suggest-tests/src/static_suggestions.rs). +Static suggestions are simple: they are just [globs](https://crates.io/crates/glob) +which map to a `x` command. In `suggest-tests`, this is implemented with a +simple `macro_rules` macro. + +### Dynamic suggestions + +Defined [here](https://github.com/rust-lang/rust/blob/master/src/tools/suggest-tests/src/dynamic_suggestions.rs). +These are more complicated than static suggestions and are implemented as +functions with the following signature: `fn(&Path) -> Vec`. In +other words, each suggestion takes a path to a modified file and (after running +arbitrary Rust code) can return any number of suggestions, or none. Dynamic +suggestions are useful for situations where fine-grained control over +suggestions is needed. For example, modifications to the `compiler/xyz/` path +should trigger the `x test compiler/xyz` suggestion. In the future, dynamic +suggestions might even read file contents to determine if (what) tests should +run. + +## Adding a suggestion + +The following steps should serve as a rough guide to add suggestions to +`suggest-tests` (very welcome!): + +1. Determine the rules for your suggestion. Is it simple and operates only on + a single path or does it match globs? Does it need fine-grained control over + the resulting command or does "one size fit all"? +2. Based on the previous step, decide if your suggestion should be implemented + as either static or dynamic. +3. Implement the suggestion. If it is dynamic then a test is highly recommended, + to verify that your logic is correct and to give an example of the suggestion. + See the [tests.rs](https://github.com/rust-lang/rust/blob/master/src/tools/suggest-tests/src/tests.rs) + file. +4. Open a PR implementing your suggestion. **(TODO: add example PR)** diff --git a/src/tests/ui.md b/src/tests/ui.md index 3556f4e23..1f899f586 100644 --- a/src/tests/ui.md +++ b/src/tests/ui.md @@ -6,18 +6,18 @@ UI tests are a particular [test suite](compiletest.md#test-suites) of compiletes ## Introduction -The tests in [`src/test/ui`] are a collection of general-purpose tests which +The tests in [`tests/ui`] are a collection of general-purpose tests which primarily focus on validating the console output of the compiler, but can be used for many other purposes. For example, tests can also be configured to [run the resulting program](#controlling-passfail-expectations) to verify its behavior. -[`src/test/ui`]: https://github.com/rust-lang/rust/blob/master/src/test/ui +[`tests/ui`]: https://github.com/rust-lang/rust/blob/master/tests/ui ## General structure of a test -A test consists of a Rust source file located anywhere in the `src/test/ui` directory. -For example, [`src/test/ui/hello.rs`] is a basic hello-world test. +A test consists of a Rust source file located anywhere in the `tests/ui` directory. +For example, [`tests/ui/hello.rs`] is a basic hello-world test. Compiletest will use `rustc` to compile the test, and compare the output against the expected output which is stored in a `.stdout` or `.stderr` file @@ -40,7 +40,7 @@ By default, a test is built as an executable binary. If you need a different crate type, you can use the `#![crate_type]` attribute to set it as needed. -[`src/test/ui/hello.rs`]: https://github.com/rust-lang/rust/blob/master/src/test/ui/hello.rs +[`tests/ui/hello.rs`]: https://github.com/rust-lang/rust/blob/master/tests/ui/hello.rs ## Output comparison @@ -84,8 +84,11 @@ check for output files: [Normalization](#normalization)). * `dont-check-compiler-stderr` — Ignores stderr from the compiler. * `dont-check-compiler-stdout` — Ignores stdout from the compiler. +* `compare-output-lines-by-subset` — Checks that the output contains the + contents of the stored output files by lines opposed to checking for strict + equality. -UI tests run with with `-Zdeduplicate-diagnostics=no` flag which disables +UI tests run with `-Zdeduplicate-diagnostics=no` flag which disables rustc's built-in diagnostic deduplication mechanism. This means you may see some duplicate messages in the output. This helps illuminate situations where duplicate diagnostics are being @@ -99,7 +102,7 @@ platforms, mainly about filenames. Compiletest makes the following replacements on the compiler output: - The directory where the test is defined is replaced with `$DIR`. - Example: `/path/to/rust/src/test/ui/error-codes` + Example: `/path/to/rust/tests/ui/error-codes` - The directory to the standard library source is replaced with `$SRC_DIR`. Example: `/path/to/rust/library` - Line and column numbers for paths in `$SRC_DIR` are replaced with `LL:COL`. @@ -160,8 +163,8 @@ The corresponding reference file will use the normalized output to test both Please see [`ui/transmute/main.rs`][mrs] and [`main.stderr`] for a concrete usage example. -[mrs]: https://github.com/rust-lang/rust/blob/master/src/test/ui/transmute/main.rs -[`main.stderr`]: https://github.com/rust-lang/rust/blob/master/src/test/ui/transmute/main.stderr +[mrs]: https://github.com/rust-lang/rust/blob/master/tests/ui/transmute/main.rs +[`main.stderr`]: https://github.com/rust-lang/rust/blob/master/tests/ui/transmute/main.stderr Besides `normalize-stderr-32bit` and `-64bit`, one may use any target information or stage supported by [`ignore-X`](headers.md#ignoring-tests) @@ -387,7 +390,7 @@ Tests with the `*-pass` headers can be overridden with the `--pass` command-line option: ```sh -./x.py test src/test/ui --pass check +./x test tests/ui --pass check ``` The `--pass` option only affects UI tests. @@ -426,30 +429,30 @@ from the internet – we often name the test after the issue plus a short description. Ideally, the test should be added to a directory that helps identify what piece of code is being tested here (e.g., -`src/test/ui/borrowck/issue-54597-reject-move-out-of-borrow-via-pat.rs`) +`tests/ui/borrowck/issue-54597-reject-move-out-of-borrow-via-pat.rs`) When writing a new feature, **create a subdirectory to store your tests**. For example, if you are implementing RFC 1234 ("Widgets"), then it might make -sense to put the tests in a directory like `src/test/ui/rfc1234-widgets/`. +sense to put the tests in a directory like `tests/ui/rfc1234-widgets/`. In other cases, there may already be a suitable directory. (The proper directory structure to use is actually an area of active debate.) -Over time, the [`src/test/ui`] directory has grown very fast. +Over time, the [`tests/ui`] directory has grown very fast. There is a check in [tidy](intro.md#tidy) that will ensure none of the subdirectories has more than 1000 entries. Having too many files causes problems because it isn't editor/IDE friendly and the GitHub UI won't show more than 1000 entries. -However, since `src/test/ui` (UI test root directory) and `src/test/ui/issues` +However, since `tests/ui` (UI test root directory) and `tests/ui/issues` directories have more than 1000 entries, we set a different limit for those directories. So, please avoid putting a new test there and try to find a more relevant place. For example, if your test is related to closures, you should put it in -`src/test/ui/closures`. +`tests/ui/closures`. If you're not sure where is the best place, it's still okay to add to -`src/test/ui/issues/`. +`tests/ui/issues/`. When you reach the limit, you could increase it by tweaking [here][ui test tidy]. @@ -482,7 +485,7 @@ Then, it applies the suggestion and compares against `.fixed` (they must match). Finally, the fixed source is compiled, and this compilation is required to succeed. Usually when creating a rustfix test you will generate the `.fixed` file -automatically with the `x.py test --bless` option. +automatically with the `x test --bless` option. The `run-rustfix` header will cause *all* suggestions to be applied, even if they are not [`MachineApplicable`](../diagnostics.md#suggestions). @@ -512,7 +515,7 @@ If in the rare case you encounter a test that has different behavior, you can run something like the following to generate the alternate stderr file: ```sh -./x.py test src/test/ui --compare-mode=polonius --bless +./x test tests/ui --compare-mode=polonius --bless ``` Currently none of the compare modes are checked in CI for UI tests. diff --git a/src/the-parser.md b/src/the-parser.md index 0d37704e8..f0436350a 100644 --- a/src/the-parser.md +++ b/src/the-parser.md @@ -1,8 +1,5 @@ # Lexing and Parsing -As of January 2021, the lexer and parser are undergoing -refactoring to allow extracting them into libraries. - The very first thing the compiler does is take the program (in Unicode characters) and turn it into something the compiler can work with more conveniently than strings. This happens in two stages: Lexing and Parsing. diff --git a/src/thir.md b/src/thir.md index 2a811be3d..2197cad71 100644 --- a/src/thir.md +++ b/src/thir.md @@ -82,7 +82,7 @@ Thir { neg: false, }, }, - // expression 1, scope surronding literal 1 + // expression 1, scope surrounding literal 1 Expr { ty: i32, temp_lifetime: Some( diff --git a/src/tracing.md b/src/tracing.md index 0bba73f74..af484ab5f 100644 --- a/src/tracing.md +++ b/src/tracing.md @@ -144,6 +144,9 @@ $ RUSTC_LOG=debug rustc +stage1 my-file.rs 2>all-log # compilers. $ RUSTC_LOG=rustc_codegen_ssa=info rustc +stage1 my-file.rs +# This will show all logs in `rustc_codegen_ssa` and `rustc_resolve`. +$ RUSTC_LOG=rustc_codegen_ssa,rustc_resolve rustc +stage1 my-file.rs + # This will show the output of all `info!` calls made by rustdoc # or any rustc library it calls. $ RUSTDOC_LOG=info rustdoc +stage1 my-file.rs diff --git a/src/traits/resolution.md b/src/traits/resolution.md index 9cf753b19..8fd4272a8 100644 --- a/src/traits/resolution.md +++ b/src/traits/resolution.md @@ -163,7 +163,7 @@ impl Get for Box { } ``` -What happens when we invoke `get_it(&Box::new(1_u16))`, for example? In this +What happens when we invoke `get(&Box::new(1_u16))`, for example? In this case, the `Self` type is `Box` – that unifies with both impls, because the first applies to all types `T`, and the second to all `Box`. In order for this to be unambiguous, the compiler does a *winnowing* @@ -247,8 +247,8 @@ In this second selection, we do not consider any where-clauses to be in scope because we know that each resolution will resolve to a particular impl. One interesting twist has to do with nested obligations. In general, in codegen, -we only to figure out which candidate applies, we do not care about nested obligations, -as these are already assumed to be true. Nonetheless, we *do* currently do fulfill all of them. +we only need to figure out which candidate applies, and we do not care about nested obligations, +as these are already assumed to be true. Nonetheless, we *do* currently fulfill all of them. That is because it can sometimes inform the results of type inference. That is, we do not have the full substitutions in terms of the type variables of the impl available to us, so we must run trait selection to figure diff --git a/src/traits/unsize.md b/src/traits/unsize.md new file mode 100644 index 000000000..b11760992 --- /dev/null +++ b/src/traits/unsize.md @@ -0,0 +1,84 @@ +# [`CoerceUnsized`](https://doc.rust-lang.org/std/ops/trait.CoerceUnsized.html) + +`CoerceUnsized` is primarily concerned with data containers. When a struct +(typically, a smart pointer) implements `CoerceUnsized`, that means that the +data it points to is being unsized. + +Some implementors of `CoerceUnsized` include: +* `&T` +* `Arc` +* `Box` + +This trait is (eventually) intended to be implemented by user-written smart +pointers, and there are rules about when a type is allowed to implement +`CoerceUnsized` that are explained in the trait's documentation. + +# [`Unsize`](https://doc.rust-lang.org/std/marker/trait.Unsize.html) + +To contrast, the `Unsize` trait is concerned the actual types that are allowed +to be unsized. + +This is not intended to be implemented by users ever, since `Unsize` does not +instruct the compiler (namely codegen) *how* to unsize a type, just whether it +is allowed to be unsized. This is paired somewhat intimately with codegen +which must understand how types are represented and unsized. + +## Primitive unsizing implementations + +Built-in implementations are provided for: +* `T` -> `dyn Trait + 'a` when `T: Trait` (and `T: Sized + 'a`, and `Trait` + is object safe). +* `[T; N]` -> `[T]` + +## Structural implementations + +There are two implementations of `Unsize` which can be thought of as +structural: +* `(A1, A2, .., An): Unsize<(A1, A2, .., U)>` given `An: Unsize`, which + allows the tail field of a tuple to be unsized. This is gated behind the + [`unsized_tuple_coercion`] feature. +* `Struct<.., Pi, .., Pj, ..>: Unsize>` given + `TailField: Unsize`, which allows the tail field of a + struct to be unsized if it is the only field that mentions generic parameters + `Pi`, .., `Pj` (which don't need to be contiguous). + +The rules for the latter implementation are slightly complicated, since they +may allow more than one parameter to be changed (not necessarily unsized) and +are best stated in terms of the tail field of the struct. + +[`unsized_tuple_coercion`]: https://doc.rust-lang.org/beta/unstable-book/language-features/unsized-tuple-coercion.html + +## Upcasting implementations + +Two things are called "upcasting" internally: +1. True upcasting `dyn SubTrait` -> `dyn SuperTrait` (this also allows + dropping auto traits and adjusting lifetimes, as below). +2. Dropping auto traits and adjusting the lifetimes of dyn trait + *without changing the principal[^1]*: + `dyn Trait + AutoTraits... + 'a` -> `dyn Trait + NewAutoTraits... + 'b` + when `AutoTraits` ⊇ `NewAutoTraits`, and `'a: 'b`. + +These may seem like different operations, since (1.) includes adjusting the +vtable of a dyn trait, while (2.) is a no-op. However, to the type system, +these are handled with much the same code. + +This built-in implementation of `Unsize` is the most involved, particularly +after [it was reworked](https://github.com/rust-lang/rust/pull/114036) to +support the complexities of associated types. + +Specifically, the upcasting algorithm involves: For each supertrait of the +source dyn trait's principal (including itself)... +1. Unify the super trait ref with the principal of the target (making sure + we only ever upcast to a true supertrait, and never [via an impl]). +2. For every auto trait in the source, check that it's present in the principal + (allowing us to drop auto traits, but never gain new ones). +3. For every projection in the source, check that it unifies with a single + projection in the target (since there may be more than one given + `trait Sub: Sup<.., A = i32> + Sup<.., A = u32>`). + +[via an impl]: https://github.com/rust-lang/rust/blob/f3457dbf84cd86d284454d12705861398ece76c3/tests/ui/traits/trait-upcasting/illegal-upcast-from-impl.rs#L19 + +Specifically, (3.) prevents a choice of projection bound to guide inference +unnecessarily, though it may guide inference when it is unambiguous. + +[^1]: The principal is the one non-auto trait of a `dyn Trait`. \ No newline at end of file diff --git a/src/turbofishing-and-early-late-bound.md b/src/turbofishing-and-early-late-bound.md new file mode 100644 index 000000000..a1d8a64e8 --- /dev/null +++ b/src/turbofishing-and-early-late-bound.md @@ -0,0 +1,120 @@ +# Turbofishing's interactions with early/late bound parameters + +The early/late bound parameter distinction on functions introduces some complications +when providing generic arguments to functions. This document discusses what those are +and how they might interact with future changes to make more things late bound. + +## Can't turbofish generic arguments on functions sometimes + +When a function has any late bound lifetime parameters (be they explicitly defined or +implicitly introduced via lifetime elision) we disallow specifying any lifetime arguments +on the function. Sometimes this is a hard error other times it is a future compat lint +([`late_bound_lifetime_arguments`](https://github.com/rust-lang/rust/issues/42868)). + +```rust +fn early<'a: 'a>(a: &'a ()) -> &'a () { a } +fn late<'a>(a: &'a ()) -> &'a () { a } + +fn mixed<'a, 'b: 'b>(a: &'a (), b: &'b ()) -> &'a () { a } + +struct Foo; +impl Foo { + fn late<'a>(self, a: &'a ()) -> &'a () { a } +} + +fn main() { + // fine + let f = early::<'static>; + + // some variation of hard errors and future compat lints + Foo.late::<'static>(&()); + let f = late::<'static>; + let f = mixed::<'static, 'static>; + let f = mixed::<'static>; + late::<'static>(&()); +} +``` + +The justification for this is that late bound parameters are not present on the +`FnDef` so the arguments to late bound parameters can't be present in the substs +for the type. i.e. the `late` function in the above code snippet would not have +any generic parameters on the `FnDef` zst: +```rust +// example desugaring of the `late` function and its zst + builtin Fn impl +struct LateFnDef; +impl<'a> Fn<(&'a ())> for LateFnDef { + type Output = &'a (); + ... +} +``` + +The cause for some situations giving future compat lints and others giving hard errors +is a little arbitrary but explainable: +- It's always a hard error for method calls +- It's only a hard error on paths to free functions if there is no unambiguous way to +create the substs for the fndef from the lifetime arguments. (i.e. the amount of +lifetimes provided must be exactly equal to the amount of early bound lifetimes or +else it's a hard error) + +## Back compat issues from turning early bound to late bound + +Because of the previously mentioned restriction on turbofishing generic arguments, it +is a breaking change to upgrade a lifetime from early bound to late bound as it can cause +existing turbofishies to become hard errors/future compat lints. + +Many t-types members have expressed interest in wanting more parameters to be late bound. +We cannot do so if making something late bound is going to break code that many would +expect to work (judging by the future compat lint issue many people do expect to be able +to turbofish late bound parameters). + +## Interactions with late bound type/const parameters + +If we were to make some type/const parameters late bound we would definitely not want +to disallow turbofishing them as it presumably(?) would break a Tonne of code. + +While lifetimes do differ from type/consts in some ways I(BoxyUwU) do not believe there +is any justification for why it would make sense to allow turbofishing late bound +type/const parameters but not late bound lifetimes. + +## Removing the hard error/fcw + +From reasons above it seems reasonable that we may want to remove the hard error and fcw +(removing the errors/fcw is definitely a blocker for making more things late bound). + +example behaviour: +```rust +fn late<'a>(a: &'a ()) -> &'a () { a } + +fn accepts_fn(_: impl for<'a> Fn(&'a ()) -> &'a ()) {} +fn accepts_fn_2(_: impl Fn(&'static ()) -> &'static ()) {} + +fn main() { + let f = late::<'static>; + + accepts_fn(f); //~ error: `f` doesnt implement `for<'a> Fn(&'a ()) -> &'a ()` + accepts_fn_2(f) // works + + accepts_fn(late) // works +} +```` + +one potential complication is that we would want a way to specify a generic argument +to a function without having to specify arguments for all previous parameters. i.e. +ideally you could write the following code somehow. +```rust +fn late<'a, 'b>(_: &'a (), _: &'b ()) {} + +fn accepts_fn(_: impl for<'a> Fn(&'a (), &'static ())) {} + +fn main() { + // a naive implementation would have a `ReInfer` as the subst for `'a` parameter + // no longer allowing the FnDef to satisfy the `for<'a> Fn(&'a ()` bound + let f = late::<'_, 'static>; + accepts_fn(f); +} +``` +Maybe we can just special case astconv for `_`/`'_` arguments for late bound parameters somehow +and have it not mean the same thing as `_` for early bound parameters. Regardless I think we +would need a solution that would allow writing the above code even if it was done by some new +syntax such as havign to write `late::` (naturally `k#no_argument` +would only make sense as an argument to late bound parameters). diff --git a/src/ty.md b/src/ty.md index 90e17e85f..b7a2ba8e4 100644 --- a/src/ty.md +++ b/src/ty.md @@ -74,9 +74,9 @@ HIR is built, some basic type inference and type checking is done. During the ty figure out what the `ty::Ty` of everything is and we also check if the type of something is ambiguous. The `ty::Ty` is then used for type checking while making sure everything has the expected type. The [`astconv` module][astconv] is where the code responsible for converting a -`rustc_hir::Ty` into a `ty::Ty` is located. This occurs during the type-checking phase, -but also in other parts of the compiler that want to ask questions like "what argument types does -this function expect?" +`rustc_hir::Ty` into a `ty::Ty` is located. The main routine used is `ast_ty_to_ty`. This occurs +during the type-checking phase, but also in other parts of the compiler that want to ask +questions like "what argument types does this function expect?" [astconv]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/astconv/index.html @@ -133,33 +133,79 @@ the `ty::Ty` to be a thin pointer-like type. This allows us to do cheap comparisons for equality, along with the other benefits of interning. -[tykind]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.TyKind.html +[tykind]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.TyKind.html ## Allocating and working with types -To allocate a new type, you can use the various `mk_` methods defined on the `tcx`. These have names +To allocate a new type, you can use the various `new_*` methods defined on +[`Ty`](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Ty.html). +These have names that correspond mostly to the various kinds of types. For example: ```rust,ignore -let array_ty = tcx.mk_array(elem_ty, len * 2); +let array_ty = Ty::new_array_with_const_len(tcx, ty, count); ``` These methods all return a `Ty<'tcx>` – note that the lifetime you get back is the lifetime of the arena that this `tcx` has access to. Types are always canonicalized and interned (so we never allocate exactly the same type twice). -> N.B. -> Because types are interned, it is possible to compare them for equality efficiently using `==` -> – however, this is almost never what you want to do unless you happen to be hashing and looking -> for duplicates. This is because often in Rust there are multiple ways to represent the same type, -> particularly once inference is involved. If you are going to be testing for type equality, you -> probably need to start looking into the inference code to do it right. - You can also find various common types in the `tcx` itself by accessing its fields: `tcx.types.bool`, `tcx.types.char`, etc. (See [`CommonTypes`] for more.) [`CommonTypes`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/context/struct.CommonTypes.html + +## Comparing types + +Because types are interned, it is possible to compare them for equality efficiently using `==` +– however, this is almost never what you want to do unless you happen to be hashing and looking +for duplicates. This is because often in Rust there are multiple ways to represent the same type, +particularly once inference is involved. + +For example, the type `{integer}` (`ty::Infer(ty::IntVar(..))` an integer inference variable, +the type of an integer literal like `0`) and `u8` (`ty::UInt(..)`) should often be treated as +equal when testing whether they can be assigned to each other (which is a common operation in +diagnostics code). `==` on them will return `false` though, since they are different types. + +The simplest way to compare two types correctly requires an inference context (`infcx`). +If you have one, you can use `infcx.can_eq(param_env, ty1, ty2)` +to check whether the types can be made equal. +This is typically what you want to check during diagnostics, which is concerned with questions such +as whether two types can be assigned to each other, not whether they're represented identically in +the compiler's type-checking layer. + +When working with an inference context, you have to be careful to ensure that potential inference +variables inside the types actually belong to that inference context. If you are in a function +that has access to an inference context already, this should be the case. Specifically, this is the +case during HIR type checking or MIR borrow checking. + +Another consideration is normalization. Two types may actually be the same, but one is behind an +associated type. To compare them correctly, you have to normalize the types first. This is +primarily a concern during HIR type checking and with all types from a `TyCtxt` query +(for example from `tcx.type_of()`). + +When a `FnCtxt` or an `ObligationCtxt` is available during type checking, `.normalize(ty)` +should be used on them to normalize the type. After type checking, diagnostics code can use +`tcx.normalize_erasing_regions(ty)`. + +There are also cases where using `==` on `Ty` is fine. This is for example the case in late lints +or after monomorphization, since type checking has been completed, meaning all inference variables +are resolved and all regions have been erased. In these cases, if you know that inference variables +or normalization won't be a concern, `#[allow]` or `#[expect]`ing the lint is recommended. + +When diagnostics code does not have access to an inference context, it should be threaded through +the function calls if one is available in some place (like during type checking). + +If no inference context is available at all, then one can be created as described in +[type-inference]. But this is only useful when the involved types (for example, if +they came from a query like `tcx.type_of()`) are actually substituted with fresh +inference variables using [`fresh_args_for_item`]. This can be used to answer questions +like "can `Vec` for any `T` be unified with `Vec`?". + +[type-inference]: ./type-inference.md#creating-an-inference-context +[`fresh_args_for_item`]: https://doc.rust-lang.org/beta/nightly-rustc/rustc_infer/infer/struct.InferCtxt.html#method.fresh_substs_for_item + ## `ty::TyKind` Variants Note: `TyKind` is **NOT** the functional programming concept of *Kind*. @@ -207,16 +253,16 @@ There are many variants on the `TyKind` enum, which you can see by looking at it - [**And many more**...][kindvars] [wikiadt]: https://en.wikipedia.org/wiki/Algebraic_data_type -[kindadt]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.TyKind.html#variant.Adt -[kindforeign]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.TyKind.html#variant.Foreign -[kindstr]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.TyKind.html#variant.Str -[kindslice]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.TyKind.html#variant.Slice -[kindarray]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.TyKind.html#variant.Array -[kindrawptr]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.TyKind.html#variant.RawPtr -[kindref]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.TyKind.html#variant.Ref -[kindparam]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.TyKind.html#variant.Param -[kinderr]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.TyKind.html#variant.Error -[kindvars]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/enum.TyKind.html#variants +[kindadt]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.TyKind.html#variant.Adt +[kindforeign]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.TyKind.html#variant.Foreign +[kindstr]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.TyKind.html#variant.Str +[kindslice]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.TyKind.html#variant.Slice +[kindarray]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.TyKind.html#variant.Array +[kindrawptr]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.TyKind.html#variant.RawPtr +[kindref]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.TyKind.html#variant.Ref +[kindparam]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.TyKind.html#variant.Param +[kinderr]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.TyKind.html#variant.Error +[kindvars]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.TyKind.html#variants ## Import conventions @@ -241,7 +287,7 @@ struct MyStruct { x: u32, y: T } The type `MyStruct` would be an instance of `TyKind::Adt`: ```rust,ignore -Adt(&'tcx AdtDef, SubstsRef<'tcx>) +Adt(&'tcx AdtDef, GenericArgsRef<'tcx>) // ------------ --------------- // (1) (2) // @@ -255,12 +301,12 @@ There are two parts: parameters. In our example, this is the `MyStruct` part *without* the argument `u32`. (Note that in the HIR, structs, enums and unions are represented differently, but in `ty::Ty`, they are all represented using `TyKind::Adt`.) -- The [`SubstsRef`][substsref] is an interned list of values that are to be substituted for the - generic parameters. In our example of `MyStruct`, we would end up with a list like `[u32]`. - We’ll dig more into generics and substitutions in a little bit. +- The [`GenericArgsRef`][GenericArgsRef] is an interned list of values that are to be substituted +for the generic parameters. In our example of `MyStruct`, we would end up with a list like +`[u32]`. We’ll dig more into generics and substitutions in a little bit. [adtdef]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.AdtDef.html -[substsref]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/subst/type.SubstsRef.html +[GenericArgsRef]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/subst/type.GenericArgsRef.html **`AdtDef` and `DefId`** @@ -317,13 +363,13 @@ delaying a redundant span bug. ## Question: Why not substitute “inside” the `AdtDef`? -Recall that we represent a generic struct with `(AdtDef, substs)`. So why bother with this scheme? +Recall that we represent a generic struct with `(AdtDef, args)`. So why bother with this scheme? Well, the alternate way we could have chosen to represent types would be to always create a new, fully-substituted form of the `AdtDef` where all the types are already substituted. This seems like -less of a hassle. However, the `(AdtDef, substs)` scheme has some advantages over this. +less of a hassle. However, the `(AdtDef, args)` scheme has some advantages over this. -First, `(AdtDef, substs)` scheme has an efficiency win: +First, `(AdtDef, args)` scheme has an efficiency win: ```rust,ignore struct MyStruct { diff --git a/src/type-checking.md b/src/type-checking.md index 08f15e30b..b60694201 100644 --- a/src/type-checking.md +++ b/src/type-checking.md @@ -1,10 +1,12 @@ # Type checking -The [`rustc_typeck`][typeck] crate contains the source for "type collection" -and "type checking", as well as a few other bits of related functionality. (It -draws heavily on the [type inference] and [trait solving].) +The [`hir_analysis`] crate contains the source for "type collection" as well +as a bunch of related functionality. +Checking the bodies of functions is implemented in the [`hir_typeck`] crate. +These crates draw heavily on the [type inference] and [trait solving]. -[typeck]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/index.html +[`hir_analysis`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_analysis/index.html +[`hir_typeck`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_hir_typeck/index.html [type inference]: ./type-inference.md [trait solving]: ./traits/resolution.md diff --git a/src/type-inference.md b/src/type-inference.md index ca88c1686..4043789d0 100644 --- a/src/type-inference.md +++ b/src/type-inference.md @@ -36,18 +36,16 @@ signature, such as the `'a` in `for<'a> fn(&'a u32)`. A region is ## Creating an inference context -You create and "enter" an inference context by doing something like +You create an inference context by doing something like the following: ```rust,ignore -tcx.infer_ctxt().enter(|infcx| { - // Use the inference context `infcx` here. -}) +let infcx = tcx.infer_ctxt().build(); +// Use the inference context `infcx` here. ``` -Within the closure, -`infcx` has the type `InferCtxt<'a, 'tcx>` for some fresh `'a`, -while `'tcx` is the same as outside the inference context. +`infcx` has the type `InferCtxt<'tcx>`, the same `'tcx` lifetime as on +the `tcx` it was built from. The `tcx.infer_ctxt` method actually returns a builder, which means there are some kinds of configuration you can do before the `infcx` is @@ -70,7 +68,7 @@ inference works, or perhaps this blog post on [Unification in the Chalk project]: http://smallcultfollowing.com/babysteps/blog/2017/03/25/unification-in-chalk-part-1/ All told, the inference context stores five kinds of inference variables -(as of June 2021): +(as of March 2023): - Type variables, which come in three varieties: - General type variables (the most common). These can be unified with any diff --git a/src/walkthrough.md b/src/walkthrough.md index 84cb1296f..b8311b19a 100644 --- a/src/walkthrough.md +++ b/src/walkthrough.md @@ -163,7 +163,7 @@ Depending on the feature/change/bug fix/improvement, implementation may be relatively-straightforward or it may be a major undertaking. You can always ask for help or mentorship from more experienced compiler devs. Also, you don't have to be the one to implement your feature; but keep in mind that if you -don't it might be a while before someone else does. +don't, it might be a while before someone else does. For the `?` macro feature, I needed to go understand the relevant parts of macro expansion in the compiler. Personally, I find that [improving the @@ -181,7 +181,7 @@ gate is removed when the feature is stabilized. make your changes/improvements. When you open a PR on the [rust-lang/rust], a bot will assign your PR to a -review. If there is a particular Rust team member you are working with, you can +reviewer. If there is a particular Rust team member you are working with, you can request that reviewer by leaving a comment on the thread with `r? @reviewer-github-id` (e.g. `r? @eddyb`). If you don't know who to request, don't request anyone; the bot will assign someone automatically based on which files you changed. diff --git a/src/what-does-early-late-bound-mean.md b/src/what-does-early-late-bound-mean.md new file mode 100644 index 000000000..9a6a0b139 --- /dev/null +++ b/src/what-does-early-late-bound-mean.md @@ -0,0 +1,199 @@ +# Early and Late Bound Parameter Definitions + +Understanding this page likely requires a rudimentary understanding of higher ranked +trait bounds/`for<'a>`and also what types such as `dyn for<'a> Trait<'a>` and + `for<'a> fn(&'a u32)` mean. Reading [the nomincon chapter](https://doc.rust-lang.org/nomicon/hrtb.html) +on HRTB may be useful for understanding this syntax. The meaning of `for<'a> fn(&'a u32)` +is incredibly similar to the meaning of `T: for<'a> Trait<'a>`. + +If you are looking for information on the `RegionKind` variants `ReLateBound` and `ReEarlyBound` +you should look at the section on [bound vars and params](./bound-vars-and-params.md). This section +discusses what makes generic parameters on functions and closures late/early bound. Not the general +concept of bound vars and generic parameters which `RegionKind` has named somewhat confusingly +with this topic. + +## What does it mean for parameters to be early or late bound + +All function definitions conceptually have a ZST (this is represented by `TyKind::FnDef` in rustc). +The only generics on this ZST are the early bound parameters of the function definition. e.g. +```rust +fn foo<'a>(_: &'a u32) {} + +fn main() { + let b = foo; + // ^ `b` has type `FnDef(foo, [])` (no args because `'a` is late bound) + assert!(std::mem::size_of_val(&b) == 0); +} +``` + +In order to call `b` the late bound parameters do need to be provided, these are inferred at the +call site instead of when we refer to `foo`. +```rust +fn main() { + let b = foo; + let a: &'static u32 = &10; + foo(a); + // the lifetime argument for `'a` on `foo` is inferred at the callsite + // the generic parameter `'a` on `foo` is inferred to `'static` here +} +``` + +Because late bound parameters are not part of the `FnDef`'s args this allows us to prove trait +bounds such as `F: for<'a> Fn(&'a u32)` where `F` is `foo`'s `FnDef`. e.g. +```rust +fn foo_early<'a, T: Trait<'a>>(_: &'a u32, _: T) {} +fn foo_late<'a, T>(_: &'a u32, _: T) {} + +fn accepts_hr_func Fn(&'a u32, u32)>(_: F) {} + +fn main() { + // doesnt work, the substituted bound is `for<'a> FnDef<'?0>: Fn(&'a u32, u32)` + // `foo_early` only implements `for<'a> FnDef<'a>: Fn(&'a u32, u32)`- the lifetime + // of the borrow in the function argument must be the same as the lifetime + // on the `FnDef`. + accepts_hr_func(foo_early); + + // works, the substituted bound is `for<'a> FnDef: Fn(&'a u32, u32)` + accepts_hr_func(foo_late); +} + +// the builtin `Fn` impls for `foo_early` and `foo_late` look something like: +// `foo_early` +impl<'a, T: Trait<'a>> Fn(&'a u32, T) for FooEarlyFnDef<'a, T> { ... } +// `foo_late` +impl<'a, T> Fn(&'a u32, T) for FooLateFnDef { ... } + +``` + +Early bound parameters are present on the `FnDef`. Late bound generic parameters are not present +on the `FnDef` but are instead constrained by the builtin `Fn*` impl. + +The same distinction applies to closures. Instead of `FnDef` we are talking about the anonymous +closure type. Closures are [currently unsound](https://github.com/rust-lang/rust/issues/84366) in +ways that are closely related to the distinction between early/late bound +parameters (more on this later) + +The early/late boundness of generic parameters is only relevent for the desugaring of +functions/closures into types with builtin `Fn*` impls. It does not make sense to talk about +in other contexts. + +The `generics_of` query in rustc only contains early bound parameters. In this way it acts more +like `generics_of(my_func)` is the generics for the FnDef than the generics provided to the function +body although it's not clear to the author of this section if this was the actual justification for +making `generics_of` behave this way. + +## What parameters are currently late bound + +Below are the current requirements for determining if a generic parameter is late bound. It is worth +keeping in mind that these are not necessarily set in stone and it is almost certainly possible to +be more flexible. + +### Must be a lifetime parameter + +Rust can't support types such as `for dyn Trait` or `for fn(T)`, this is a +fundamental limitation of the language as we are required to monomorphize type/const +parameters and cannot do so behind dynamic dispatch. (technically we could probably +support `for dyn MarkerTrait` as there is nothing to monomorphize) + +Not being able to support `for dyn Trait` resulted in making all type and const +parameters early bound. Only lifetime parameters can be late bound. + +### Must not appear in the where clauses + +In order for a generic parameter to be late bound it must not appear in any where clauses. +This is currently an incredibly simplistic check that causes lifetimes to be early bound even +if the where clause they appear in are always true, or implied by well formedness of function +arguments. e.g. +```rust +fn foo1<'a: 'a>(_: &'a u32) {} +// ^^ early bound parameter because it's in a `'a: 'a` clause +// even though the bound obviously holds all the time +fn foo2<'a, T: Trait<'a>(a: T, b: &'a u32) {} +// ^^ early bound parameter because it's used in the `T: Trait<'a>` clause +fn foo3<'a, T: 'a>(_: &'a T) {} +// ^^ early bound parameter because it's used in the `T: 'a` clause +// even though that bound is implied by wellformedness of `&'a T` +fn foo4<'a, 'b: 'a>(_: Inv<&'a ()>, _: Inv<&'b ()>) {} +// ^^ ^^ ^^^ note: +// ^^ ^^ `Inv` stands for `Invariant` and is used to +// ^^ ^^ make the the type parameter invariant. This +// ^^ ^^ is necessary for demonstration purposes as +// ^^ ^^ `for<'a, 'b> fn(&'a (), &'b ())` and +// ^^ ^^ `for<'a> fn(&'a u32, &'a u32)` are subtypes- +// ^^ ^^ of eachother which makes the bound trivially +// ^^ ^^ satisfiable when making the fnptr. `Inv` +// ^^ ^^ disables this subtyping. +// ^^ ^^ +// ^^^^^^ both early bound parameters because they are present in the +// `'b: 'a` clause +``` + +The reason for this requirement is that we cannot represent the `T: Trait<'a>` or `'a: 'b` clauses +on a function pointer. `for<'a, 'b> fn(Inv<&'a ()>, Inv<&'b ()>)` is not a valid function pointer to +represent`foo4` as it would allow calling the function without `'b: 'a` holding. + +### Must be constrained by where clauses or function argument types + +The builtin impls of the `Fn*` traits for closures and `FnDef`s cannot not have any unconstrained +parameters. For example the following impl is illegal: +```rust +impl<'a> Trait for u32 { type Assoc = &'a u32; } +``` +We must not end up with a similar impl for the `Fn*` traits e.g. +```rust +impl<'a> Fn<()> for FnDef { type Assoc = &'a u32 } +``` + +Violating this rule can trivially lead to unsoundness as seen in [#84366](https://github.com/rust-lang/rust/issues/84366). +Additionally if we ever support late bound type params then an impl like: +```rust +impl Fn<()> for FnDef { type Assoc = T; } +``` +would break the compiler in various ways. + +In order to ensure that everything functions correctly, we do not allow generic parameters to +be late bound if it would result in a builtin impl that does not constrain all of the generic +parameters on the builtin impl. Making a generic parameter be early bound trivially makes it be +constrained by the builtin impl as it ends up on the self type. + +Because of the requirement that late bound parameters must not appear in where clauses, checking +this is simpler than the rules for checking impl headers constrain all the parameters on the impl. +We only have to ensure that all late bound parameters appear at least once in the function argument +types outside of an alias (e.g. an associated type). + +The requirement that they not indirectly be in the args of an alias for it to count is the +same as why the follow code is forbidden: +```rust +impl OtherTrait for ::Assoc { type Assoc = T } +``` +There is no guarantee that `::Assoc` will normalize to different types for every +instantiation of `T`. If we were to allow this impl we could get overlapping impls and the +same is true of the builtin `Fn*` impls. + +## Making more generic parameters late bound + +It is generally considered desirable for more parameters to be late bound as it makes +the builtin `Fn*` impls more flexible. Right now many of the requirements for making +a parameter late bound are overly restrictive as they are tied to what we can currently +(or can ever) do with fn ptrs. + +It would be theoretically possible to support late bound params in `where`-clauses in the +language by introducing implication types which would allow us to express types such as: +`for<'a, 'b: 'a> fn(Inv<&'a u32>, Inv<&'b u32>)` which would ensure `'b: 'a` is upheld when +calling the function pointer. + +It would also be theoretically possible to support it by making the coercion to a fn ptr +instantiate the parameter with an infer var while still allowing the FnDef to not have the +generic parameter present as trait impls are perfectly capable of representing the where clauses +on the function on the impl itself. This would also allow us to support late bound type/const +vars allowing bounds like `F: for Fn(T)` to hold. + +It is almost somewhat unclear if we can change the `Fn` traits to be structured differently +so that we never have to make a parameter early bound just to make the builtin impl have all +generics be constrained. Of all the possible causes of a generic parameter being early bound +this seems the most difficult to remove. + +Whether these would be good ideas to implement is a separate question- they are only brought +up to illustrate that the current rules are not necessarily set in stone and a result of +"its the only way of doing this". +