Skip to content

Commit 9f3235a

Browse files
sharkdpsharkdp
authored
[red-knot] Expand test corpus (astral-sh#14360)
## Summary - Add 383 files from `crates/ruff_python_parser/resources` to the test corpus - Add 1296 files from `crates/ruff_linter/resources` to the test corpus - Use in-memory file system for tests - Improve test isolation by cleaning the test environment between checks - Add a mechanism for "known failures". Mark ~80 files as known failures. - The corpus test is now a lot slower (6 seconds). Note: While `red_knot` as a command line tool can run over all of these files without panicking, we still have a lot of test failures caused by explicitly "pulling" all types. ## Test Plan Run `cargo test -p red_knot_workspace` while making sure that - Introducing code that is known to lead to a panic fails the test - Removing code that is known to lead to a panic from `KNOWN_FAILURES`-files also fails the test
1 parent 62d6502 commit 9f3235a

5 files changed

Lines changed: 204 additions & 33 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/red_knot_workspace/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ tracing = { workspace = true }
3030

3131
[dev-dependencies]
3232
ruff_db = { workspace = true, features = ["testing"] }
33-
tempfile = { workspace = true }
33+
glob = { workspace = true }
3434

3535
[features]
3636
default = ["zstd"]

‎crates/red_knot_workspace/tests/check.rs‎

Lines changed: 194 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,122 @@
1-
use std::fs;
2-
use std::path::PathBuf;
3-
41
use red_knot_python_semantic::{HasTy, SemanticModel};
52
use red_knot_workspace::db::RootDatabase;
63
use red_knot_workspace::workspace::WorkspaceMetadata;
74
use ruff_db::files::{system_path_to_file, File};
85
use ruff_db::parsed::parsed_module;
9-
use ruff_db::system::{OsSystem, SystemPath, SystemPathBuf};
6+
use ruff_db::system::{SystemPath, SystemPathBuf, TestSystem};
107
use ruff_python_ast::visitor::source_order;
118
use ruff_python_ast::visitor::source_order::SourceOrderVisitor;
129
use ruff_python_ast::{self as ast, Alias, Expr, Parameter, ParameterWithDefault, Stmt};
1310

14-
fn setup_db(workspace_root: &SystemPath) -> anyhow::Result<RootDatabase> {
15-
let system = OsSystem::new(workspace_root);
11+
fn setup_db(workspace_root: &SystemPath, system: TestSystem) -> anyhow::Result<RootDatabase> {
1612
let workspace = WorkspaceMetadata::from_path(workspace_root, &system, None)?;
1713
RootDatabase::new(workspace, system)
1814
}
1915

20-
/// Test that all snippets in testcorpus can be checked without panic
16+
fn get_workspace_root() -> anyhow::Result<SystemPathBuf> {
17+
Ok(SystemPathBuf::from(String::from_utf8(
18+
std::process::Command::new("cargo")
19+
.args(["locate-project", "--workspace", "--message-format", "plain"])
20+
.output()?
21+
.stdout,
22+
)?)
23+
.parent()
24+
.unwrap()
25+
.to_owned())
26+
}
27+
28+
/// Test that all snippets in testcorpus can be checked without panic (except for [`KNOWN_FAILURES`])
2129
#[test]
2230
#[allow(clippy::print_stdout)]
2331
fn corpus_no_panic() -> anyhow::Result<()> {
24-
let root = SystemPathBuf::from_path_buf(tempfile::TempDir::new()?.into_path()).unwrap();
25-
let db = setup_db(&root)?;
26-
27-
let corpus = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("resources/test/corpus");
28-
29-
for path in fs::read_dir(&corpus)? {
30-
let source = path?.path();
31-
println!("checking {source:?}");
32-
let source_fn = source.file_name().unwrap().to_str().unwrap();
33-
let py_dest = root.join(source_fn);
34-
fs::copy(&source, py_dest.as_std_path())?;
35-
// this test is only asserting that we can pull every expression type without a panic
36-
// (and some non-expressions that clearly define a single type)
37-
let file = system_path_to_file(&db, py_dest).unwrap();
38-
pull_types(&db, file);
39-
40-
// try the file as a stub also
41-
println!("re-checking as .pyi");
42-
let pyi_dest = root.join(format!("{source_fn}i"));
43-
std::fs::copy(source, pyi_dest.as_std_path())?;
44-
let file = system_path_to_file(&db, pyi_dest).unwrap();
45-
pull_types(&db, file);
32+
let root = SystemPathBuf::from("/src");
33+
34+
let system = TestSystem::default();
35+
let memory_fs = system.memory_file_system();
36+
memory_fs.create_directory_all(root.as_ref())?;
37+
38+
let mut db = setup_db(&root, system.clone())?;
39+
40+
let crate_root = String::from(env!("CARGO_MANIFEST_DIR"));
41+
let workspace_root = get_workspace_root()?;
42+
let workspace_root = workspace_root.to_string();
43+
44+
let corpus = vec![
45+
format!("{crate_root}/resources/test/corpus/**/*.py"),
46+
format!("{workspace_root}/crates/ruff_python_parser/resources/**/*.py"),
47+
format!("{workspace_root}/crates/ruff_linter/resources/**/*.py"),
48+
// TODO: Enable running over typeshed stubs once there are fewer failures:
49+
// format!("{workspace_root}/crates/red_knot_vendored/vendor/typeshed/**/*.pyi"),
50+
]
51+
.into_iter()
52+
.flat_map(|pattern| glob::glob(&pattern).unwrap());
53+
54+
for path in corpus {
55+
let path = path?;
56+
let relative_path = path.strip_prefix(&workspace_root)?;
57+
58+
let (py_expected_to_fail, pyi_expected_to_fail) = KNOWN_FAILURES
59+
.iter()
60+
.find_map(|(path, py_fail, pyi_fail)| {
61+
if Some(*path)
62+
== relative_path
63+
.to_str()
64+
.map(|p| p.replace('\\', "/"))
65+
.as_deref()
66+
{
67+
Some((*py_fail, *pyi_fail))
68+
} else {
69+
None
70+
}
71+
})
72+
.unwrap_or((false, false));
73+
74+
let source = path.as_path();
75+
let source_filename = source.file_name().unwrap().to_str().unwrap();
76+
77+
let code = std::fs::read_to_string(source)?;
78+
79+
let mut check_with_file_name = |path: &SystemPath| {
80+
memory_fs.write_file(path, &code).unwrap();
81+
File::sync_path(&mut db, path);
82+
83+
// this test is only asserting that we can pull every expression type without a panic
84+
// (and some non-expressions that clearly define a single type)
85+
let file = system_path_to_file(&db, path).unwrap();
86+
87+
let result = std::panic::catch_unwind(|| pull_types(&db, file));
88+
89+
let expected_to_fail = if path.extension().map(|e| e == "pyi").unwrap_or(false) {
90+
pyi_expected_to_fail
91+
} else {
92+
py_expected_to_fail
93+
};
94+
if let Err(err) = result {
95+
if !expected_to_fail {
96+
println!("Check failed for {relative_path:?}. Consider fixing it or adding it to KNOWN_FAILURES");
97+
std::panic::resume_unwind(err);
98+
}
99+
} else {
100+
assert!(!expected_to_fail, "Expected to panic, but did not. Consider removing this path from KNOWN_FAILURES");
101+
}
102+
103+
memory_fs.remove_all();
104+
file.sync(&mut db);
105+
};
106+
107+
if source.extension().map(|e| e == "pyi").unwrap_or(false) {
108+
println!("checking {relative_path:?}");
109+
let pyi_dest = root.join(source_filename);
110+
check_with_file_name(&pyi_dest);
111+
} else {
112+
println!("checking {relative_path:?}");
113+
let py_dest = root.join(source_filename);
114+
check_with_file_name(&py_dest);
115+
116+
let pyi_dest = root.join(format!("{source_filename}i"));
117+
println!("re-checking as stub file: {pyi_dest:?}");
118+
check_with_file_name(&pyi_dest);
119+
}
46120
}
47121
Ok(())
48122
}
@@ -144,3 +218,94 @@ impl SourceOrderVisitor<'_> for PullTypesVisitor<'_> {
144218
source_order::walk_alias(self, alias);
145219
}
146220
}
221+
222+
/// Whether or not the .py/.pyi version of this file is expected to fail
223+
const KNOWN_FAILURES: &[(&str, bool, bool)] = &[
224+
// Probably related to missing support for type aliases / type params:
225+
("crates/ruff_python_parser/resources/inline/err/type_param_invalid_bound_expr.py", true, true),
226+
("crates/ruff_python_parser/resources/inline/err/type_param_type_var_invalid_default_expr.py", true, true),
227+
("crates/ruff_python_parser/resources/inline/err/type_param_param_spec_invalid_default_expr.py", true, true),
228+
("crates/ruff_python_parser/resources/inline/err/type_param_type_var_missing_default.py", true, true),
229+
("crates/ruff_python_parser/resources/inline/err/type_param_type_var_tuple_invalid_default_expr.py", true, true),
230+
("crates/ruff_python_parser/resources/inline/ok/type_param_param_spec.py", true, true),
231+
("crates/ruff_python_parser/resources/inline/ok/type_param_type_var.py", true, true),
232+
("crates/ruff_python_parser/resources/inline/ok/type_param_type_var_tuple.py", true, true),
233+
("crates/ruff_python_parser/resources/valid/statement/type.py", true, true),
234+
// Fails for unknown reasons:
235+
("crates/ruff_python_parser/resources/valid/expressions/f_string.py", true, true),
236+
("crates/ruff_linter/resources/test/fixtures/flake8_future_annotations/no_future_import_uses_union_inner.py", true, true),
237+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI011.py", true, true),
238+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI015.py", true, true),
239+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI016.py", true, true),
240+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI019.py", true, true),
241+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI020.py", true, true),
242+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI020.py", true, true),
243+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI030.py", true, true),
244+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI030.py", true, true),
245+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI034.py", true, true),
246+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI034.py", true, true),
247+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI035.py", true, true),
248+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI035.py", true, true),
249+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI036.py", true, true),
250+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI036.py", true, true),
251+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI041.py", true, true),
252+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI041.py", true, true),
253+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI051.py", true, true),
254+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI051.py", true, true),
255+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI052.py", true, true),
256+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI052.py", true, true),
257+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI055.py", true, true),
258+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI055.py", true, true),
259+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI062.py", true, true),
260+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI062.py", true, true),
261+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI063.py", true, true),
262+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI063.py", true, true),
263+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI064.py", true, true),
264+
("crates/ruff_linter/resources/test/fixtures/flake8_pyi/PYI064.py", true, true),
265+
("crates/ruff_linter/resources/test/fixtures/flake8_type_checking/TCH004_13.py", true, true),
266+
("crates/ruff_linter/resources/test/fixtures/flake8_type_checking/TCH004_13.py", true, true),
267+
("crates/ruff_linter/resources/test/fixtures/flake8_type_checking/TCH004_15.py", true, true),
268+
("crates/ruff_linter/resources/test/fixtures/flake8_type_checking/TCH004_15.py", true, true),
269+
("crates/ruff_linter/resources/test/fixtures/flake8_type_checking/quote.py", true, true),
270+
("crates/ruff_linter/resources/test/fixtures/flake8_type_checking/quote.py", true, true),
271+
("crates/ruff_linter/resources/test/fixtures/flake8_type_checking/quote2.py", true, true),
272+
("crates/ruff_linter/resources/test/fixtures/flake8_type_checking/quote2.py", true, true),
273+
("crates/ruff_linter/resources/test/fixtures/flake8_type_checking/quote3.py", true, true),
274+
("crates/ruff_linter/resources/test/fixtures/flake8_type_checking/quote3.py", true, true),
275+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F401_19.py", true, true),
276+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F401_19.py", true, true),
277+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F541.py", true, true),
278+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F541.py", true, true),
279+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F632.py", true, true),
280+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F632.py", true, true),
281+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F811_19.py", true, false),
282+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F821_0.py", true, true),
283+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F821_0.py", true, true),
284+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F821_14.py", false, true),
285+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F821_15.py", true, true),
286+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F821_15.py", true, true),
287+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F821_17.py", true, true),
288+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F821_17.py", true, true),
289+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F821_2.py", true, true),
290+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F821_2.py", true, true),
291+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F821_20.py", true, true),
292+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F821_20.py", true, true),
293+
("crates/ruff_linter/resources/test/fixtures/pyflakes/F821_26.py", true, false),
294+
("crates/ruff_linter/resources/test/fixtures/pyflakes/project/foo/bar.py", true, true),
295+
("crates/ruff_linter/resources/test/fixtures/pyflakes/project/foo/bar.py", true, true),
296+
("crates/ruff_linter/resources/test/fixtures/pyflakes/project/foo/bop/baz.py", true, true),
297+
("crates/ruff_linter/resources/test/fixtures/pyflakes/project/foo/bop/baz.py", true, true),
298+
("crates/ruff_linter/resources/test/fixtures/pylint/single_string_slots.py", true, true),
299+
("crates/ruff_linter/resources/test/fixtures/pylint/single_string_slots.py", true, true),
300+
("crates/ruff_linter/resources/test/fixtures/pyupgrade/UP037_0.py", true, true),
301+
("crates/ruff_linter/resources/test/fixtures/pyupgrade/UP037_0.py", true, true),
302+
("crates/ruff_linter/resources/test/fixtures/pyupgrade/UP039.py", true, false),
303+
("crates/ruff_linter/resources/test/fixtures/pyupgrade/UP044.py", true, true),
304+
("crates/ruff_linter/resources/test/fixtures/pyupgrade/UP044.py", true, true),
305+
("crates/ruff_linter/resources/test/fixtures/ruff/RUF013_0.py", true, true),
306+
("crates/ruff_linter/resources/test/fixtures/ruff/RUF013_0.py", true, true),
307+
("crates/ruff_linter/resources/test/fixtures/ruff/RUF013_3.py", true, true),
308+
("crates/ruff_linter/resources/test/fixtures/ruff/RUF013_3.py", true, true),
309+
("crates/ruff_linter/resources/test/fixtures/ruff/RUF022.py", true, true),
310+
("crates/ruff_linter/resources/test/fixtures/ruff/RUF022.py", true, true),
311+
];

‎crates/ruff_db/src/system/path.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,12 @@ impl From<&str> for SystemPathBuf {
491491
}
492492
}
493493

494+
impl From<String> for SystemPathBuf {
495+
fn from(value: String) -> Self {
496+
SystemPathBuf::from_utf8_path_buf(Utf8PathBuf::from(value))
497+
}
498+
}
499+
494500
impl Default for SystemPathBuf {
495501
fn default() -> Self {
496502
Self::new()

‎crates/ruff_db/src/system/test.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use super::walk_directory::WalkDirectoryBuilder;
2121
///
2222
/// ## Warning
2323
/// Don't use this system for production code. It's intended for testing only.
24-
#[derive(Default, Debug)]
24+
#[derive(Default, Debug, Clone)]
2525
pub struct TestSystem {
2626
inner: TestSystemInner,
2727
}
@@ -229,7 +229,7 @@ pub trait DbWithTestSystem: Db + Sized {
229229
}
230230
}
231231

232-
#[derive(Debug)]
232+
#[derive(Debug, Clone)]
233233
enum TestSystemInner {
234234
Stub(MemoryFileSystem),
235235
System(Arc<dyn System + RefUnwindSafe + Send + Sync>),

0 commit comments

Comments
 (0)