Skip to content

Commit

Permalink
Use an empty vendored file system in Ruff (#13436)
Browse files Browse the repository at this point in the history
## Summary

This PR changes removes the typeshed stubs from the vendored file system
shipped with ruff
and instead ships an empty "typeshed".

Making the typeshed files optional required extracting the typshed files
into a new `ruff_vendored` crate. I do like this even if all our builds
always include typeshed because it means `red_knot_python_semantic`
contains less code that needs compiling.

This also allows us to use deflate because the compression algorithm
doesn't matter for an archive containing a single, empty file.

## Test Plan

`cargo test`

I verified with ` cargo tree -f "{p} {f}" -p <package> ` that:

* red_knot_wasm: enables `deflate` compression
* red_knot: enables `zstd` compression
* `ruff`: uses stored


I'm not quiet sure how to build the binary that maturin builds but
comparing the release artifact size with `strip = true` shows a `1.5MB`
size reduction

---------

Co-authored-by: Charlie Marsh <charlie.r.marsh@gmail.com>
  • Loading branch information
MichaReiser and charliermarsh authored Sep 21, 2024
1 parent 8921fbb commit 653c090
Show file tree
Hide file tree
Showing 602 changed files with 158 additions and 126 deletions.
14 changes: 7 additions & 7 deletions .github/workflows/sync_typeshed.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,13 +37,13 @@ jobs:
- name: Sync typeshed
id: sync
run: |
rm -rf ruff/crates/red_knot_python_semantic/vendor/typeshed
mkdir ruff/crates/red_knot_python_semantic/vendor/typeshed
cp typeshed/README.md ruff/crates/red_knot_python_semantic/vendor/typeshed
cp typeshed/LICENSE ruff/crates/red_knot_python_semantic/vendor/typeshed
cp -r typeshed/stdlib ruff/crates/red_knot_python_semantic/vendor/typeshed/stdlib
rm -rf ruff/crates/red_knot_python_semantic/vendor/typeshed/stdlib/@tests
git -C typeshed rev-parse HEAD > ruff/crates/red_knot_python_semantic/vendor/typeshed/source_commit.txt
rm -rf ruff/crates/ruff_vendored/vendor/typeshed
mkdir ruff/crates/ruff_vendored/vendor/typeshed
cp typeshed/README.md ruff/crates/ruff_vendored/vendor/typeshed
cp typeshed/LICENSE ruff/crates/ruff_vendored/vendor/typeshed
cp -r typeshed/stdlib ruff/crates/ruff_vendored/vendor/typeshed/stdlib
rm -rf ruff/crates/ruff_vendored/vendor/typeshed/stdlib/@tests
git -C typeshed rev-parse HEAD > ruff/crates/ruff_vendored/vendor/typeshed/source_commit.txt
- name: Commit the changes
id: commit
if: ${{ steps.sync.outcome == 'success' }}
Expand Down
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ fail_fast: true

exclude: |
(?x)^(
crates/red_knot_python_semantic/vendor/.*|
crates/ruff_vendored/vendor/.*|
crates/red_knot_workspace/resources/.*|
crates/ruff_linter/resources/.*|
crates/ruff_linter/src/rules/.*/snapshots/.*|
Expand Down
19 changes: 15 additions & 4 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 3 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ license = "MIT"
[workspace.dependencies]
ruff = { path = "crates/ruff" }
ruff_cache = { path = "crates/ruff_cache" }
ruff_db = { path = "crates/ruff_db" }
ruff_db = { path = "crates/ruff_db", default-features = false }
ruff_diagnostics = { path = "crates/ruff_diagnostics" }
ruff_formatter = { path = "crates/ruff_formatter" }
ruff_graph = { path = "crates/ruff_graph" }
Expand All @@ -34,11 +34,12 @@ ruff_python_trivia = { path = "crates/ruff_python_trivia" }
ruff_server = { path = "crates/ruff_server" }
ruff_source_file = { path = "crates/ruff_source_file" }
ruff_text_size = { path = "crates/ruff_text_size" }
ruff_vendored = { path = "crates/ruff_vendored" }
ruff_workspace = { path = "crates/ruff_workspace" }

red_knot_python_semantic = { path = "crates/red_knot_python_semantic" }
red_knot_server = { path = "crates/red_knot_server" }
red_knot_workspace = { path = "crates/red_knot_workspace" }
red_knot_workspace = { path = "crates/red_knot_workspace", default-features = false }

aho-corasick = { version = "1.1.3" }
annotate-snippets = { version = "0.9.2", features = ["color"] }
Expand Down
3 changes: 1 addition & 2 deletions crates/red_knot/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,8 @@ license.workspace = true

[dependencies]
red_knot_python_semantic = { workspace = true }
red_knot_workspace = { workspace = true }
red_knot_workspace = { workspace = true, features = ["zstd"] }
red_knot_server = { workspace = true }

ruff_db = { workspace = true, features = ["os", "cache"] }

anyhow = { workspace = true }
Expand Down
9 changes: 1 addition & 8 deletions crates/red_knot_python_semantic/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ bitflags = { workspace = true }
camino = { workspace = true }
compact_str = { workspace = true }
countme = { workspace = true }
once_cell = { workspace = true }
ordermap = { workspace = true }
salsa = { workspace = true }
thiserror = { workspace = true }
Expand All @@ -35,20 +34,14 @@ smallvec = { workspace = true }
static_assertions = { workspace = true }
test-case = { workspace = true }

[build-dependencies]
path-slash = { workspace = true }
walkdir = { workspace = true }
zip = { workspace = true, features = ["zstd", "deflate"] }

[dev-dependencies]
ruff_db = { workspace = true, features = ["os", "testing"] }
ruff_python_parser = { workspace = true }
ruff_vendored = { workspace = true }

anyhow = { workspace = true }
insta = { workspace = true }
tempfile = { workspace = true }
walkdir = { workspace = true }
zip = { workspace = true }

[lints]
workspace = true
Expand Down
3 changes: 1 addition & 2 deletions crates/red_knot_python_semantic/src/db.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ pub trait Db: SourceDb + Upcast<dyn SourceDb> {
pub(crate) mod tests {
use std::sync::Arc;

use crate::module_resolver::vendored_typeshed_stubs;
use ruff_db::files::{File, Files};
use ruff_db::system::{DbWithTestSystem, System, TestSystem};
use ruff_db::vendored::VendoredFileSystem;
Expand All @@ -33,7 +32,7 @@ pub(crate) mod tests {
Self {
storage: salsa::Storage::default(),
system: TestSystem::default(),
vendored: vendored_typeshed_stubs().clone(),
vendored: ruff_vendored::file_system().clone(),
events: std::sync::Arc::default(),
files: Files::default(),
}
Expand Down
4 changes: 1 addition & 3 deletions crates/red_knot_python_semantic/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,7 @@ use rustc_hash::FxHasher;

pub use db::Db;
pub use module_name::ModuleName;
pub use module_resolver::{
resolve_module, system_module_search_paths, vendored_typeshed_stubs, Module,
};
pub use module_resolver::{resolve_module, system_module_search_paths, Module};
pub use program::{Program, ProgramSettings, SearchPathSettings, SitePackages};
pub use python_version::PythonVersion;
pub use semantic_model::{HasTy, SemanticModel};
Expand Down
1 change: 0 additions & 1 deletion crates/red_knot_python_semantic/src/module_resolver/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ pub use module::Module;
pub use resolver::resolve_module;
pub(crate) use resolver::{file_to_module, SearchPaths};
use ruff_db::system::SystemPath;
pub use typeshed::vendored_typeshed_stubs;

use crate::module_resolver::resolver::search_paths;
use crate::Db;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,8 @@ mod tests {
fn typeshed_versions_consistent_with_vendored_stubs() {
let db = TestDb::new();
let vendored_typeshed_versions = vendored_typeshed_versions(&db);
let vendored_typeshed_dir = Path::new("vendor/typeshed").canonicalize().unwrap();
let vendored_typeshed_dir =
Path::new(env!("CARGO_MANIFEST_DIR")).join("../ruff_vendored/vendor/typeshed");

let mut empty_iterator = true;

Expand Down

This file was deleted.

9 changes: 6 additions & 3 deletions crates/red_knot_python_semantic/src/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -701,12 +701,12 @@ enum CallOutcome<'db> {

impl<'db> CallOutcome<'db> {
/// Create a new `CallOutcome::Callable` with given return type.
fn callable(return_ty: Type<'db>) -> CallOutcome {
fn callable(return_ty: Type<'db>) -> CallOutcome<'db> {
CallOutcome::Callable { return_ty }
}

/// Create a new `CallOutcome::NotCallable` with given not-callable type.
fn not_callable(not_callable_ty: Type<'db>) -> CallOutcome {
fn not_callable(not_callable_ty: Type<'db>) -> CallOutcome<'db> {
CallOutcome::NotCallable { not_callable_ty }
}

Expand All @@ -719,7 +719,10 @@ impl<'db> CallOutcome<'db> {
}

/// Create a new `CallOutcome::Union` with given wrapped outcomes.
fn union(called_ty: Type<'db>, outcomes: impl Into<Box<[CallOutcome<'db>]>>) -> CallOutcome {
fn union(
called_ty: Type<'db>,
outcomes: impl Into<Box<[CallOutcome<'db>]>>,
) -> CallOutcome<'db> {
CallOutcome::Union {
called_ty,
outcomes: outcomes.into(),
Expand Down
4 changes: 2 additions & 2 deletions crates/red_knot_wasm/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ default = ["console_error_panic_hook"]

[dependencies]
red_knot_python_semantic = { workspace = true }
red_knot_workspace = { workspace = true }
red_knot_workspace = { workspace = true, default-features = false, features = ["deflate"] }

ruff_db = { workspace = true }
ruff_db = { workspace = true, features = [] }
ruff_notebook = { workspace = true }

console_error_panic_hook = { workspace = true, optional = true }
Expand Down
6 changes: 6 additions & 0 deletions crates/red_knot_workspace/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ ruff_cache = { workspace = true }
ruff_db = { workspace = true, features = ["os", "cache"] }
ruff_python_ast = { workspace = true }
ruff_text_size = { workspace = true }
ruff_vendored = { workspace = true }

anyhow = { workspace = true }
crossbeam = { workspace = true }
Expand All @@ -31,5 +32,10 @@ tracing = { workspace = true }
ruff_db = { workspace = true, features = ["testing"] }
tempfile = { workspace = true }

[features]
default = ["zstd"]
zstd = ["ruff_vendored/zstd"]
deflate = ["ruff_vendored/deflate"]

[lints]
workspace = true
8 changes: 4 additions & 4 deletions crates/red_knot_workspace/src/db.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use std::sync::Arc;
use salsa::plumbing::ZalsaDatabase;
use salsa::{Cancelled, Event};

use red_knot_python_semantic::{vendored_typeshed_stubs, Db as SemanticDb, Program};
use red_knot_python_semantic::{Db as SemanticDb, Program};
use ruff_db::files::{File, Files};
use ruff_db::system::System;
use ruff_db::vendored::VendoredFileSystem;
Expand Down Expand Up @@ -124,7 +124,7 @@ impl SemanticDb for RootDatabase {
#[salsa::db]
impl SourceDb for RootDatabase {
fn vendored(&self) -> &VendoredFileSystem {
vendored_typeshed_stubs()
ruff_vendored::file_system()
}

fn system(&self) -> &dyn System {
Expand Down Expand Up @@ -161,7 +161,7 @@ pub(crate) mod tests {

use salsa::Event;

use red_knot_python_semantic::{vendored_typeshed_stubs, Db as SemanticDb};
use red_knot_python_semantic::Db as SemanticDb;
use ruff_db::files::Files;
use ruff_db::system::{DbWithTestSystem, System, TestSystem};
use ruff_db::vendored::VendoredFileSystem;
Expand All @@ -183,7 +183,7 @@ pub(crate) mod tests {
Self {
storage: salsa::Storage::default(),
system: TestSystem::default(),
vendored: vendored_typeshed_stubs().clone(),
vendored: ruff_vendored::file_system().clone(),
files: Files::default(),
events: Arc::default(),
}
Expand Down
2 changes: 1 addition & 1 deletion crates/ruff/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ default-run = "ruff"

[dependencies]
ruff_cache = { workspace = true }
ruff_db = { workspace = true }
ruff_db = { workspace = true, default-features = false, features = ["os"] }
ruff_diagnostics = { workspace = true }
ruff_graph = { workspace = true, features = ["serde", "clap"] }
ruff_linter = { workspace = true, features = ["clap"] }
Expand Down
6 changes: 2 additions & 4 deletions crates/ruff_db/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,17 @@ tracing = { workspace = true }
tracing-subscriber = { workspace = true, optional = true }
tracing-tree = { workspace = true, optional = true }
rustc-hash = { workspace = true }

[target.'cfg(not(target_arch="wasm32"))'.dependencies]
zip = { workspace = true, features = ["zstd"] }
zip = { workspace = true }

[target.'cfg(target_arch="wasm32")'.dependencies]
web-time = { version = "1.1.0" }
zip = { workspace = true, features = ["deflate"] }

[dev-dependencies]
insta = { workspace = true }
tempfile = { workspace = true }

[features]
default = ["os"]
cache = ["ruff_cache"]
os = ["ignore"]
serde = ["dep:serde", "camino/serde1"]
Expand Down
5 changes: 3 additions & 2 deletions crates/ruff_db/src/files.rs
Original file line number Diff line number Diff line change
Expand Up @@ -503,7 +503,8 @@ mod tests {
use crate::files::{system_path_to_file, vendored_path_to_file, FileError};
use crate::system::DbWithTestSystem;
use crate::tests::TestDb;
use crate::vendored::tests::VendoredFileSystemBuilder;
use crate::vendored::VendoredFileSystemBuilder;
use zip::CompressionMethod;

#[test]
fn system_existing_file() -> crate::system::Result<()> {
Expand Down Expand Up @@ -548,7 +549,7 @@ mod tests {
fn stubbed_vendored_file() -> crate::system::Result<()> {
let mut db = TestDb::new();

let mut vendored_builder = VendoredFileSystemBuilder::new();
let mut vendored_builder = VendoredFileSystemBuilder::new(CompressionMethod::Stored);
vendored_builder
.add_file("test.pyi", "def foo() -> str")
.unwrap();
Expand Down
5 changes: 3 additions & 2 deletions crates/ruff_db/src/parsed.rs
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,9 @@ mod tests {
use crate::parsed::parsed_module;
use crate::system::{DbWithTestSystem, SystemPath, SystemVirtualPath};
use crate::tests::TestDb;
use crate::vendored::{tests::VendoredFileSystemBuilder, VendoredPath};
use crate::vendored::{VendoredFileSystemBuilder, VendoredPath};
use crate::Db;
use zip::CompressionMethod;

#[test]
fn python_file() -> crate::system::Result<()> {
Expand Down Expand Up @@ -150,7 +151,7 @@ mod tests {
fn vendored_file() {
let mut db = TestDb::new();

let mut vendored_builder = VendoredFileSystemBuilder::new();
let mut vendored_builder = VendoredFileSystemBuilder::new(CompressionMethod::Stored);
vendored_builder
.add_file(
"path.pyi",
Expand Down
Loading

0 comments on commit 653c090

Please sign in to comment.