Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2,340 changes: 1,303 additions & 1,037 deletions Cargo.lock

Large diffs are not rendered by default.

24 changes: 14 additions & 10 deletions cli/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ version = "7.1.0"
authors = ["Phylum, Inc. <engineering@phylum.io>"]
license = "GPL-3.0-or-later"
edition = "2021"
rust-version = "1.76.0"
rust-version = "1.80.0"
autotests = false

[[test]]
Expand All @@ -21,17 +21,17 @@ end-to-end-tests = ["extensions"]
[dependencies]
anyhow = "1.0.44"
axum = "0.7.4"
base64 = "0.21.1"
base64 = "0.22.1"
bytes = "1.1.0"
chrono = { version = "^0.4", default-features = false, features = ["serde", "clock"] }
cidr = "0.2.0"
clap = { version = "4.0.9", features = ["string", "wrap_help"] }
console = "0.15.2"
dashmap = "6.0.1"
deno_ast = { version = "0.38.1", features = ["transpiling"], optional = true }
deno_core = { version = "0.280.0", optional = true }
deno_runtime = { version = "0.159.0", optional = true }
dialoguer = { version = "0.10.0", features = ["fuzzy-select"] }
deno_ast = { version = "0.41.2", features = ["transpiling"], optional = true }
deno_core = { version = "0.307.0", optional = true }
deno_runtime = { version = "0.177.0", optional = true }
dialoguer = { version = "0.11.0", features = ["fuzzy-select"] }
env_logger = "0.10.0"
futures = "^0.3"
git2 = { version = "0.19.0", default-features = false }
Expand All @@ -50,7 +50,7 @@ phylum_types = { git = "https://github.com/phylum-dev/phylum-types", branch = "d
prettytable-rs = "0.10.0"
rand = "0.8.4"
regex = "1.5.5"
reqwest = { version = "0.11.3", features = ["blocking", "json", "rustls-tls", "rustls-tls-native-roots", "rustls-tls-webpki-roots"], default-features = false }
reqwest = { version = "0.12.7", features = ["blocking", "json", "rustls-tls", "rustls-tls-native-roots", "rustls-tls-webpki-roots"], default-features = false }
rsa = { version = "0.9.2", features = ["sha2"] }
serde_json = "1.0.85"
serde = { version = "1.0.144", features = ["derive"] }
Expand All @@ -61,8 +61,8 @@ tempfile = "3.3.0"
textwrap = "0.16.0"
thiserror = "1.0.29"
tokio = { version = "^1.0", features = ["full"] }
toml = "0.7.4"
unicode-width = "0.1.9"
toml = "0.8.19"
unicode-width = "0.2.0"
url = { version = "2", features = ["serde"] }
uuid = { version = "1.4.1", features = ["v4"] }
vuln-reach = { git = "https://github.com/phylum-dev/vuln-reach", optional = true }
Expand All @@ -73,7 +73,11 @@ zip = { version = "2.1.0", default-features = false, features = ["deflate"] }
[target.'cfg(unix)'.dependencies]
birdcage = { version = "0.8.1" }

[target.'cfg(windows)'.dependencies]
# HACK: The Win32_Security feature is required by deno_io, but their crate didn't enable it
windows-sys = { version = "0.52.0", features = ["Win32_Security"] }

[dev-dependencies]
assert_cmd = "2.0.4"
predicates = { version = "3.0", default-features = false, features = ["diff"] }
wiremock = "0.5.7"
wiremock = "0.6.2"
2 changes: 1 addition & 1 deletion cli/src/api/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -636,7 +636,7 @@ mod tests {
let mut guard = responder_token_holder.lock().unwrap();
let auth_header = HeaderName::from_str("Authorization").unwrap();

*guard = request.headers.get(&auth_header).map(|v| v.as_str().to_owned());
*guard = request.headers.get(&auth_header).map(|v| v.to_str().unwrap().to_owned());

ResponseTemplate::new(200)
.set_body_string(r#"{"job_id": "59482a54-423b-448d-8325-f171c9dc336b"}"#)
Expand Down
4 changes: 2 additions & 2 deletions cli/src/commands/extensions/api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ use std::rc::Rc;
use std::str::FromStr;

use anyhow::{anyhow, Error, Result};
use deno_runtime::deno_core::{op2, OpDecl, OpState};
use deno_runtime::permissions::PermissionsContainer;
use deno_core::{op2, OpDecl, OpState};
use deno_runtime::deno_permissions::PermissionsContainer;
use phylum_lockfile::ParsedLockfile;
use phylum_project::ProjectConfig;
use phylum_types::types::auth::{AccessToken, RefreshToken};
Expand Down
3 changes: 1 addition & 2 deletions cli/src/commands/extensions/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -220,8 +220,7 @@ async fn handle_install_extension(
}

fn ask_overwrite(extension: &Extension) -> Result<()> {
let mut prompt = Confirm::new();
prompt
let prompt = Confirm::new()
.with_prompt(format!(
"Another version of the '{}' extension is already installed. Overwrite?",
extension.name()
Expand Down
2 changes: 1 addition & 1 deletion cli/src/commands/extensions/state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::ops::Deref;
use std::rc::Rc;

use anyhow::{anyhow, Result};
use deno_runtime::deno_core::OpState;
use deno_core::OpState;
use futures::future::BoxFuture;
use tokio::sync::OnceCell;

Expand Down
41 changes: 19 additions & 22 deletions cli/src/commands/init.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! Subcommand `phylum init`.

use std::path::Path;
use std::{env, io, iter};
use std::{env, iter};

use anyhow::Context;
use clap::parser::ValuesRef;
Expand Down Expand Up @@ -122,25 +122,20 @@ async fn prompt_project(
}

/// Ask for the desired project name.
fn prompt_project_name() -> io::Result<String> {
fn prompt_project_name() -> dialoguer::Result<String> {
// Use directory name as default project name.
let current_dir = env::current_dir()?;
let default_name = current_dir.file_name().and_then(|name| name.to_str());

let mut prompt = Input::new();

// Suggest default if we found a directory name.
//
// NOTE: We don't use dialoguer's built-in default here so we can add a more
// explicit `default` label.
match default_name {
Some(default_name) => {
prompt.with_prompt(format!("Project Name [default: {default_name}]"));
prompt.allow_empty(true);
},
None => {
let _ = prompt.with_prompt("Project Name");
},
let prompt = match default_name {
Some(default_name) => Input::new()
.with_prompt(format!("Project Name [default: {default_name}]"))
.allow_empty(true),
None => Input::new().with_prompt("Project Name"),
};

let mut name: String = prompt.interact_text()?;
Expand Down Expand Up @@ -207,7 +202,7 @@ async fn prompt_repository_url() -> anyhow::Result<Option<String>> {
fn prompt_depfiles(
cli_depfiles: Option<ValuesRef<'_, String>>,
cli_depfile_type: Option<&String>,
) -> io::Result<Vec<DepfileConfig>> {
) -> dialoguer::Result<Vec<DepfileConfig>> {
// Prompt for dependency files if they weren't specified.
let depfiles = match cli_depfiles {
Some(depfiles) => depfiles.cloned().collect(),
Expand All @@ -226,7 +221,7 @@ fn prompt_depfiles(
}

/// Ask for the dependency file names.
fn prompt_depfile_names() -> io::Result<Vec<String>> {
fn prompt_depfile_names() -> dialoguer::Result<Vec<String>> {
// Find all known dependency files below the current directory.
let mut depfiles = phylum_lockfile::find_depfiles_at(".")
.iter()
Expand Down Expand Up @@ -272,19 +267,18 @@ fn prompt_depfile_names() -> io::Result<Vec<String>> {
}

// Construct dialoguer freetext prompt.
let mut input = Input::new();
if prompt {
input.with_prompt("Other dependency files (comma separated paths)");
let mut input = if prompt {
Input::new().with_prompt("Other dependency files (comma separated paths)")
} else {
input.with_prompt(
Input::new().with_prompt(
"No known dependency files found in the current directory.\nDependency files (comma \
separated paths)",
);
)
};

// Allow empty as escape hatch if people already selected a valid dependency
// file.
input.allow_empty(!depfiles.is_empty());
input = input.allow_empty(!depfiles.is_empty());

// Prompt for additional files.
let other_depfiles: String = input.interact_text()?;
Expand All @@ -301,7 +295,10 @@ fn prompt_depfile_names() -> io::Result<Vec<String>> {
}

/// Find the type for a dependency file path.
fn find_depfile_type(depfile: &str, cli_depfile_type: Option<&String>) -> io::Result<String> {
fn find_depfile_type(
depfile: &str,
cli_depfile_type: Option<&String>,
) -> dialoguer::Result<String> {
if let Some(cli_depfile_type) = cli_depfile_type {
// Use CLI dependency file type if specified.
return Ok(cli_depfile_type.into());
Expand All @@ -326,7 +323,7 @@ fn find_depfile_type(depfile: &str, cli_depfile_type: Option<&String>) -> io::Re
}

/// Ask for the dependency file type.
fn prompt_depfile_type(depfile: &str, mut formats: Vec<&str>) -> io::Result<String> {
fn prompt_depfile_type(depfile: &str, mut formats: Vec<&str>) -> dialoguer::Result<String> {
// Allow all formats if no matching formats were found.
if formats.is_empty() {
formats = LockfileFormat::iter().map(|format| format.name()).collect();
Expand Down
73 changes: 30 additions & 43 deletions cli/src/deno.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,21 @@
use std::borrow::Cow;
use std::path::{Path, PathBuf};
use std::rc::Rc;
use std::sync::Arc;

use anyhow::{anyhow, Context, Error, Result};
use console::style;
use dashmap::DashMap;
use deno_ast::{
EmitOptions, EmittedSource, MediaType, ParseParams, SourceMapOption, SourceTextInfo,
TranspileOptions,
EmitOptions, EmittedSourceBytes, MediaType, ParseParams, SourceMapOption, TranspileOptions,
};
use deno_core::error::JsError;
use deno_core::{
include_js_files, ExtensionFileSource, ModuleCodeString, ModuleLoadResponse, ModuleSourceCode,
RequestedModuleType,
include_js_files, Extension, ExtensionFileSource, ModuleCodeString, ModuleLoadResponse,
ModuleLoader, ModuleSource, ModuleSourceCode, ModuleSpecifier, ModuleType, RequestedModuleType,
ResolutionKind,
};
use deno_runtime::deno_core::error::JsError;
use deno_runtime::deno_core::{
Extension, ModuleLoader, ModuleSource, ModuleSpecifier, ModuleType, ResolutionKind,
SourceMapGetter,
};
use deno_runtime::permissions::{Permissions, PermissionsContainer, PermissionsOptions};
use deno_runtime::deno_permissions::{Permissions, PermissionsContainer, PermissionsOptions};
use deno_runtime::worker::{MainWorker, WorkerOptions};
use deno_runtime::{fmt_errors, BootstrapOptions};
use futures::future::BoxFuture;
Expand Down Expand Up @@ -66,22 +63,19 @@ pub async fn run(
..Default::default()
};

let main_module =
deno_core::resolve_path(&extension.entry_point().to_string_lossy(), &PathBuf::from("."))?;
let main_module = deno_core::resolve_path(extension.entry_point(), &PathBuf::from("."))?;

let bootstrap =
BootstrapOptions { args, user_agent: "phylum-cli/extension".into(), ..Default::default() };

let module_loader = Rc::new(ExtensionsModuleLoader::new(extension.path()));
let source_map_getter: Rc<dyn SourceMapGetter> = Rc::new(module_loader.clone());

let origin_storage_dir = extension.state_path();

let options = WorkerOptions {
origin_storage_dir,
module_loader,
bootstrap,
source_map_getter: Some(source_map_getter),
extensions: vec![phylum_api],
..Default::default()
};
Expand Down Expand Up @@ -219,7 +213,7 @@ impl ModuleLoader for ExtensionsModuleLoader {
// Load either a local file under the extensions directory, or a Deno standard
// library module. Reject all URLs that do not fit these two use
// cases.
let mut code = match module_specifier.scheme() {
let code = match module_specifier.scheme() {
"file" => {
ExtensionsModuleLoader::load_from_filesystem(&extension_path, &module_specifier)
.await?
Expand All @@ -228,31 +222,30 @@ impl ModuleLoader for ExtensionsModuleLoader {
_ => return Err(anyhow!("Unsupported module specifier: {}", module_specifier)),
};

if should_transpile {
let transpiled =
source_mapper.transpile(module_specifier.to_string(), &code, media_type)?;
code.clone_from(&transpiled.text);
let module_source = if should_transpile {
let transpiled = source_mapper.transpile(
module_specifier.to_string(),
code.into(),
media_type,
)?;
ModuleSourceCode::Bytes(transpiled.source.into_boxed_slice().into())
} else {
source_mapper.source_cache.insert(module_specifier.to_string(), code.clone());
}
source_mapper
.source_cache
.insert(module_specifier.to_string(), code.clone().into());
ModuleSourceCode::String(code.into())
};

Ok(ModuleSource::new(
module_type,
ModuleSourceCode::String(code.into()),
&module_specifier,
None,
))
Ok(ModuleSource::new(module_type, module_source, &module_specifier, None))
}))
}
}

impl SourceMapGetter for ExtensionsModuleLoader {
fn get_source_map(&self, file_name: &str) -> Option<Vec<u8>> {
let transpiled = self.source_mapper.transpiled_cache.get(file_name)?;
transpiled.source_map.clone().map(|map| map.into_bytes())
transpiled.source_map.clone()
}

fn get_source_line(&self, file_name: &str, line_number: usize) -> Option<String> {
fn get_source_mapped_source_line(&self, file_name: &str, line_number: usize) -> Option<String> {
let source = self.source_mapper.source_cache.get(file_name)?;
source.lines().nth(line_number).map(|line| line.to_owned())
}
Expand All @@ -261,8 +254,8 @@ impl SourceMapGetter for ExtensionsModuleLoader {
/// Module source map cache.
#[derive(Default)]
struct SourceMapper {
transpiled_cache: DashMap<String, EmittedSource>,
source_cache: DashMap<String, String>,
transpiled_cache: DashMap<String, EmittedSourceBytes>,
source_cache: DashMap<String, Arc<str>>,
}

impl SourceMapper {
Expand All @@ -274,21 +267,19 @@ impl SourceMapper {
fn transpile(
&self,
specifier: impl Into<String>,
code: impl Into<String>,
code: Arc<str>,
media_type: MediaType,
) -> Result<EmittedSource> {
) -> Result<EmittedSourceBytes> {
let specifier = specifier.into();

// Load module if it is not in the cache.
if !self.transpiled_cache.contains_key(&specifier) {
let code = code.into();

// Add the original code to the cache.
self.source_cache.insert(specifier.clone(), code.clone());

// Parse module.
let parsed = deno_ast::parse_module(ParseParams {
text_info: SourceTextInfo::from_string(code),
text: code,
specifier: specifier.parse()?,
capture_tokens: false,
scope_analysis: false,
Expand All @@ -306,12 +297,8 @@ impl SourceMapper {
self.transpiled_cache.insert(specifier.clone(), transpiled);
}

// Clone fields manually, since derive is missing.
let transpiled = self.transpiled_cache.get(&specifier).unwrap();
Ok(EmittedSource {
source_map: transpiled.source_map.clone(),
text: transpiled.text.clone(),
})
Ok(transpiled.clone())
}
}

Expand Down
2 changes: 1 addition & 1 deletion cli/src/permissions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use birdcage::error::Result as SandboxResult;
#[cfg(unix)]
use birdcage::{Birdcage, Exception, Sandbox};
#[cfg(feature = "extensions")]
use deno_runtime::permissions::PermissionsOptions;
use deno_runtime::deno_permissions::PermissionsOptions;
use serde::de::Error as _;
use serde::{Deserialize, Deserializer, Serialize};

Expand Down
2 changes: 1 addition & 1 deletion cli/tests/extensions/permissions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ fn incorrect_net_permission_unsuccessful_run() {
test_cli
.run(["incorrect-net-perms"])
.failure()
.stderr("❗ Error: Requires net access to \"phylum.io\"\n");
.stderr("❗ Error: Requires net access to \"phylum.io:443\"\n");
}

#[test]
Expand Down
2 changes: 1 addition & 1 deletion lockfile/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ serde = { version = "1.0.144", features = ["derive"] }
serde_json = "1.0.85"
serde_yaml = "0.9.2"
thiserror = "1.0.40"
toml = "0.7.4"
toml = "0.8.19"
urlencoding = "2.1.2"
walkdir = "2.3.2"

Expand Down