Merge branch 'main' into debuginfo-cache

Author: ranweiler

docs/troubleshooting.md

@@ -0,0 +1,53 @@
+# Troubleshooting issues with binaries
+
+The agent exposes some validation tools to help troubleshoot issues with a binary.
+This allows the user to debug and fix errors that could prevent a job from running.
+
+## Using the validation tools
+Download a zip file containing the agent binaries:
+
+```
+onefuzz tools get <destination_folder>
+```
+
+Extract the zip file in a folder of your choice.
+Navigate to the folder that matches your os.
+Run the following command to see the tools available:
+`onefuzz-agent.exe validate --help`
+The current list of commands are:
+   - run_setup : Run the setup script
+   - validate_libfuzzer:  Validate the libfuzzer target
+   - execution_log: Get the execution logs to debug loading issues
+
+   More tools might be added in the future so please refer the help command to get the most up to date list.
+
+
+## In a docker container
+
+It could also be helpful to run the those command in an environment to closely match the vm where the agent is deployed.
+A docker container can help with that scenario.
+
+Make sure [docker](https://docs.docker.com/desktop/) is installed and runs properly.
+
+Navigate to the folder that matches your os in the tools folder created earlier and build the docker container:
+
+```cmd
+docker build --t <container_name> .
+```
+
+Use the container interactively to execute the validation command:
+
+windows
+
+```
+docker run --it --rm --entrypoint powershell <image_name>
+```
+
+linux
+
+```
+docker run --it --rm --entrypoint bash <image_name>
+```
+
+From there you can navigate to the onefuzz directory and execute the validation commands
+

docs/unmanaged-nodes.md

@@ -75,7 +75,7 @@ Under the `client_credential` section of the agent config file, update `client_i
 
 Save the config to the file.
 
-### Start the agent.
+### Start the agent
 
 Navigate to the folder corresponding to your OS.
 Set the necessary environment variable by running the script `set-env.ps1` (for Windows) or `set-env.sh` (for Linux).
@@ -111,7 +111,7 @@ From here you will be able to schedule jobs on that pool and they will run.
 
 ## Troubleshooting
 
-### increase the verbosity of the logs
+### Increase the verbosity of the logs
 
 It can help when investigating issues to increase the log verbosity. you will need to set the [RUST_LOG](https://docs.rs/env_logger/latest/env_logger/#enabling-logging) environment variable when starting docker
 
@@ -127,7 +127,7 @@ log_level can be any of
 - debug
 - trace
 
-### use the container interactively
+### Use the container interactively
 
 you can use the container interactively by with the following command
 
@@ -143,7 +143,7 @@ linux
 docker run --it --rm --entrypoint bash <image_name>
 ```
 
-### mount a local folder in the container
+### Mount a local folder in the container
 
 docker allows you to [mount](https://docs.docker.com/storage/bind-mounts/#mount-into-a-non-empty-directory-on-the-container) a local folder when running a container
 

src/agent/onefuzz-agent/src/validations.rs

@@ -11,9 +11,9 @@ use uuid::Uuid;
 pub enum ValidationCommand {
     /// Run the setup script
     RunSetup { setup_folder: PathBuf },
-    /// Validate the libfuzzer target
+    /// Validate the libfuzzer target by attempting to run the target by itself and with some of the supplied seeds if provided
     ValidateLibfuzzer(ValidationConfig),
-    /// Get the execution logs to debug loading issues
+    /// Get the execution logs to debug dll loading issues
     ExecutionLog(ValidationConfig),
 }
 
@@ -30,8 +30,6 @@ where
         .find('=')
         .ok_or_else(|| format!("invalid KEY=value: no `=` found in `{s}`"))?;
 
-    println!("******** pos: {}", pos);
-
     Ok((s[..pos].parse()?, s[pos + 1..].parse()?))
 }
 

src/agent/onefuzz-task/src/tasks/config.rs

@@ -228,12 +228,15 @@ impl Config {
         match self {
             Config::GenericGenerator(c) => {
                 event!(task_start; EventData::Type = event_type, EventData::ToolName = c.generator_exe.clone());
+                metric!(task_start; 1.0; EventData::Type = event_type, EventData::ToolName = c.generator_exe.clone());
             }
             Config::GenericAnalysis(c) => {
                 event!(task_start; EventData::Type = event_type, EventData::ToolName = c.analyzer_exe.clone());
+                metric!(task_start; 1.0; EventData::Type = event_type, EventData::ToolName = c.analyzer_exe.clone());
             }
             _ => {
                 event!(task_start; EventData::Type = event_type);
+                metric!(task_start; 1.0; EventData::Type = event_type);
             }
         }
     }

src/agent/onefuzz-task/src/tasks/coverage/generic.rs

@@ -376,6 +376,7 @@ impl<'a> TaskContext<'a> {
                     if entry.file_type().await?.is_file() {
                         if let Err(e) = self.record_input(&entry.path()).await {
                             event!(coverage_failed; EventData::Path = entry.path().display().to_string());
+                            metric!(coverage_failed; 1.0; EventData::Path = entry.path().display().to_string());
                             warn!(
                                 "ignoring error recording coverage for input: {}, error: {}",
                                 entry.path().display(),
@@ -407,6 +408,7 @@ impl<'a> TaskContext<'a> {
 
         let s = CoverageStats::new(&self.coverage);
         event!(coverage_data; Covered = s.covered, Features = s.features, Rate = s.rate);
+        metric!(coverage_data; 1.0; Covered = s.covered, Features = s.features, Rate = s.rate);
 
         Ok(())
     }

src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs

@@ -383,6 +383,12 @@ impl TotalStats {
             EventData::Count = self.count,
             EventData::ExecsSecond = self.execs_sec
         );
+        metric!(
+            runtime_stats;
+            1.0;
+            EventData::Count = self.count,
+            EventData::ExecsSecond = self.execs_sec
+        );
     }
 }
 

src/agent/onefuzz-task/src/tasks/report/crash_report.rs

@@ -124,7 +124,8 @@ impl RegressionReport {
         };
 
         if upload_or_save_local(&self, &name, regression_reports).await? {
-            event!(event; EventData::Path = name);
+            event!(event; EventData::Path = name.clone());
+            metric!(event; 1.0; EventData::Path = name.clone());
         }
         Ok(())
     }
@@ -155,14 +156,16 @@ impl CrashTestResult {
                 if let Some(unique_reports) = unique_reports {
                     let name = report.unique_blob_name();
                     if upload_or_save_local(&report, &name, unique_reports).await? {
-                        event!(new_unique_report; EventData::Path = name);
+                        event!(new_unique_report; EventData::Path = report.unique_blob_name());
+                        metric!(new_unique_report; 1.0; EventData::Path = report.unique_blob_name());
                     }
                 }
 
                 if let Some(reports) = reports {
                     let name = report.blob_name();
                     if upload_or_save_local(&report, &name, reports).await? {
-                        event!(new_report; EventData::Path = name);
+                        event!(new_report; EventData::Path = report.blob_name());
+                        metric!(new_report; 1.0; EventData::Path = report.blob_name());
                     }
                 }
             }
@@ -171,7 +174,8 @@ impl CrashTestResult {
                 if let Some(no_repro) = no_repro {
                     let name = report.blob_name();
                     if upload_or_save_local(&report, &name, no_repro).await? {
-                        event!(new_unable_to_reproduce; EventData::Path = name);
+                        event!(new_unable_to_reproduce; EventData::Path = report.blob_name());
+                        metric!(new_unable_to_reproduce; 1.0; EventData::Path = report.blob_name());
                     }
                 }
             }

src/agent/onefuzz-telemetry/src/lib.rs

@@ -515,6 +515,33 @@ pub fn track_event(event: &Event, properties: &[EventData]) {
     try_broadcast_event(chrono::Utc::now(), event, properties);
 }
 
+pub fn track_metric(metric: &Event, value: f64, properties: &[EventData]) {
+    use appinsights::telemetry::Telemetry;
+
+    if let Some(client) = client(ClientType::Instance) {
+        let mut mtr = appinsights::telemetry::MetricTelemetry::new(metric.as_str(), value);
+        let props = mtr.properties_mut();
+        for property in properties {
+            let (name, val) = property.as_values();
+            props.insert(name.to_string(), val);
+        }
+        client.track(mtr);
+    }
+
+    if let Some(client) = client(ClientType::Microsoft) {
+        let mut mtr = appinsights::telemetry::MetricTelemetry::new(metric.as_str(), value);
+        let props = mtr.properties_mut();
+
+        for property in properties {
+            if property.can_share_with_microsoft() {
+                let (name, val) = property.as_values();
+                props.insert(name.to_string(), val);
+            }
+        }
+        client.track(mtr);
+    }
+}
+
 pub fn to_log_level(level: &appinsights::telemetry::SeverityLevel) -> log::Level {
     match level {
         Verbose => log::Level::Debug,
@@ -551,6 +578,27 @@ macro_rules! event {
     }};
 }
 
+#[macro_export]
+macro_rules! log_metrics {
+    ($name: expr; $value: expr; $metrics: expr) => {{
+        onefuzz_telemetry::track_metric(&$name, $value, &$metrics);
+    }};
+}
+
+#[macro_export]
+macro_rules! metric {
+    ($name: expr ; $value: expr ; $($k: path = $v: expr),*) => {{
+        let mut metrics = Vec::new();
+
+        $({
+            metrics.push($k(From::from($v)));
+
+        })*;
+
+        log_metrics!($name; $value; metrics);
+    }};
+}
+
 #[macro_export]
 macro_rules! log {
     ($level: expr, $($arg: tt)+) => {{
@@ -597,11 +645,3 @@ macro_rules! critical {
         onefuzz_telemetry::log!(onefuzz_telemetry::Critical, $($arg)+);
     }}
 }
-
-#[macro_export]
-macro_rules! metric {
-    ($name: expr, $value: expr) => {{
-        let client = onefuzz_telemetry::client(onefuzz_telemetry::ClientType::Instance);
-        client.track_metric($name.into(), $value);
-    }};
-}

src/agent/onefuzz/src/syncdir.rs

@@ -253,19 +253,21 @@ impl SyncedDir {
                 let file_name = item
                     .file_name()
                     .ok_or_else(|| anyhow!("invalid file path"))?;
-                let file_name_str = file_name.to_string_lossy();
+                let file_name_event_str = file_name.to_string_lossy();
+                let file_name_str_metric_str = file_name.to_string_lossy();
 
                 // explicitly ignore azcopy temporary files
                 // https://github.com/Azure/azure-storage-azcopy/blob/main/ste/xfer-remoteToLocal-file.go#L35
-                if file_name_str.starts_with(".azDownload-") {
+                if file_name_event_str.starts_with(".azDownload-") {
                     continue;
                 }
 
-                if ignore_dotfiles && file_name_str.starts_with('.') {
+                if ignore_dotfiles && file_name_event_str.starts_with('.') {
                     continue;
                 }
 
-                event!(event.clone(); EventData::Path = file_name_str);
+                event!(event.clone(); EventData::Path = file_name_event_str);
+                metric!(event.clone(); 1.0; EventData::Path = file_name_str_metric_str);
                 let destination = path.join(file_name);
                 if let Err(err) = fs::copy(&item, &destination).await {
                     let error_message = format!(
@@ -288,19 +290,21 @@ impl SyncedDir {
                 let file_name = item
                     .file_name()
                     .ok_or_else(|| anyhow!("invalid file path"))?;
-                let file_name_str = file_name.to_string_lossy();
+                let file_name_event_str = file_name.to_string_lossy();
+                let file_name_str_metric_str = file_name.to_string_lossy();
 
                 // explicitly ignore azcopy temporary files
                 // https://github.com/Azure/azure-storage-azcopy/blob/main/ste/xfer-remoteToLocal-file.go#L35
-                if file_name_str.starts_with(".azDownload-") {
+                if file_name_event_str.starts_with(".azDownload-") {
                     continue;
                 }
 
-                if ignore_dotfiles && file_name_str.starts_with('.') {
+                if ignore_dotfiles && file_name_event_str.starts_with('.') {
                     continue;
                 }
 
-                event!(event.clone(); EventData::Path = file_name_str);
+                event!(event.clone(); EventData::Path = file_name_event_str);
+                metric!(event.clone(); 1.0; EventData::Path = file_name_str_metric_str);
                 if let Err(err) = uploader.upload(item.clone()).await {
                     let error_message = format!(
                         "Couldn't upload file.  path:{} dir:{} err:{:?}",