Bug fixes and documentation (microsoft#2694)

Author: chkeita

docs/unmnaged-nodes.md

@@ -0,0 +1,80 @@
+# Unmanaged Nodes
+The default mode of OneFuzz is to run the agents inside scalesets managed by the the Onefuzz instance. But it is possible to run outside of the Instance infrastructure.
+This is the unmanaged scenario. In this mode, the user can use their own resource to participate in the fuzzing.
+
+## Set-up
+These are the steps to run an unmanaged node
+
+
+### Create an Application Registration in Azure Active Directory
+We will create the authentication method for the unmanaged node.
+From the [azure cli](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) create a new **application registration**:
+```cmd
+az ad app create --display-name <registration_name>
+```
+Then use the application `app_id` in the result to create the associated **service principal**:
+
+```cmd
+az ad sp create --id <app_id>
+```
+Take note of the `id` returned by this request. We will call it the `principal_id`.
+
+Next, create a `client_secret`:
+
+```
+az ad app credential reset --id <pp_id> --append
+```
+Take note of the `password` returned.
+
+### Authorize the application in OneFuzz
+From the OneFuzz `deployment` folder run the following script using the `app_id` from above:
+``` cmd
+python .\deploylib\registration.py register_app <onefuzz_instance_id> <subscription_id> --app_id <app_id> --role UnmanagedNode
+```
+
+### Create an unmanaged pool
+Using the OneFuzz CLI:
+``` cmd
+onefuzz pools create <pool_name> <os> --unmanaged --object_id <principal_id>
+```
+
+### Download the agent binaries and the agent configuration
+Download a zip file containing the agent binaries:
+```
+onefuzz tools get <destination_folder>
+```
+Extract the zip file in a folder of your choice.
+
+Download the configuration file for the agent:
+
+```
+onefuzz pools get_config <pool_name>
+```
+
+Under the `client_credential` section of the agent config file, update `client_id` and `client_secret`:
+```json
+{
+    "client_id": "<app_id>",
+    "client_secret": "<password>",
+}
+```
+Save the config to the file.
+
+### Start the agent.
+Navigate to the folder corresponding to your OS.
+Set the necessary environment variable by running the script `set-env.ps1` (for Windows) or `set-env.sh` (for Linux).
+Run the agent with the following command. If you need more nodes use a different `machine_guid` for each one:
+```cmd
+onefuzz-agent run --machine_id <machine_guid> -c <path_to_config_file> --reset_lock
+```
+
+### Verify that the agent is registered to OneFuzz
+
+Using the OneFuzz CLI run the following command:
+
+```
+onefuzz nodes get <machine_guid>
+```
+
+This should return one entry. Verify that the `pool_name` matched the pool name created earlier.
+From here you will be able to schedule jobs on that pool and they will be running.

src/ApiService/ApiService/Functions/AgentRegistration.cs

@@ -152,7 +152,9 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
             MachineId: machineId,
             ScalesetId: scalesetId,
             InstanceId: instanceId,
-            Version: version
+            Version: version,
+            Os: os ?? pool.Os,
+            Managed: pool.Managed
             );
 
         var r = await _context.NodeOperations.Replace(node);

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -110,7 +110,8 @@ public record Node
 
     bool ReimageRequested = false,
     bool DeleteRequested = false,
-    bool DebugKeepNode = false
+    bool DebugKeepNode = false,
+    bool Managed = true
 ) : StatefulEntityBase<NodeState>(State) {
 
     public List<NodeTasks>? Tasks { get; set; }

src/ApiService/ApiService/ServiceConfiguration.cs

@@ -32,6 +32,7 @@ public interface IServiceConfig {
     public ResourceIdentifier? OneFuzzFuncStorage { get; }
     public string? OneFuzzInstance { get; }
     public string? OneFuzzInstanceName { get; }
+    public string? OneFuzzEndpoint { get; }
     public string? OneFuzzKeyvault { get; }
 
     public string? OneFuzzMonitor { get; }
@@ -117,6 +118,7 @@ public ResourceIdentifier? OneFuzzFuncStorage {
 
     public string? OneFuzzInstance { get => GetEnv("ONEFUZZ_INSTANCE"); }
     public string? OneFuzzInstanceName { get => GetEnv("ONEFUZZ_INSTANCE_NAME"); }
+    public string? OneFuzzEndpoint { get => GetEnv("ONEFUZZ_ENDPOINT"); }
     public string? OneFuzzKeyvault { get => GetEnv("ONEFUZZ_KEYVAULT"); }
     public string? OneFuzzMonitor { get => GetEnv("ONEFUZZ_MONITOR"); }
     public string? OneFuzzOwner { get => GetEnv("ONEFUZZ_OWNER"); }

src/ApiService/ApiService/onefuzzlib/Creds.cs

@@ -105,8 +105,10 @@ public Async.Task<Region> GetBaseRegion() {
         });
     }
 
-    public Uri GetInstanceUrl()
-        => new($"https://{GetInstanceName()}.azurewebsites.net");
+    public Uri GetInstanceUrl() {
+        var onefuzzEndpoint = _config.OneFuzzEndpoint;
+        return onefuzzEndpoint != null ? new Uri(onefuzzEndpoint) : new($"https://{GetInstanceName()}.azurewebsites.net");
+    }
 
     public record ScaleSetIdentity(string principalId);
 

src/ApiService/ApiService/onefuzzlib/NodeOperations.cs

@@ -337,6 +337,10 @@ public async Async.Task CleanupBusyNodesWithoutWork() {
     }
 
     public async Async.Task<Node> ToReimage(Node node, bool done = false) {
+        if (!node.Managed) {
+            _logTracer.Info($"skip reimage for unmanaged node: {node.MachineId:Tag:MachineId}");
+            return node;
+        }
 
         var nodeState = node.State;
         if (done) {

src/ApiService/IntegrationTests/Fakes/TestServiceConfiguration.cs

@@ -31,6 +31,8 @@ public TestServiceConfiguration(string tablePrefix) {
 
     // -- Remainder not implemented --
 
+    public string? OneFuzzEndpoint => throw new System.NotImplementedException();
+
     public LogDestination[] LogDestinations { get => throw new System.NotImplementedException(); set => throw new System.NotImplementedException(); }
 
     public SeverityLevel LogSeverityLevel => throw new System.NotImplementedException();

src/agent/onefuzz-agent/src/config.rs

@@ -80,7 +80,7 @@ struct RawStaticConfig {
 }
 
 impl StaticConfig {
-    pub async fn new(data: &[u8]) -> Result<Self> {
+    pub async fn new(data: &[u8], machine_identity: Option<MachineIdentity>) -> Result<Self> {
         let config: RawStaticConfig = serde_json::from_slice(data)?;
 
         let credentials = match config.client_credentials {
@@ -104,7 +104,7 @@ impl StaticConfig {
                 managed.into()
             }
         };
-        let machine_identity = match config.machine_identity {
+        let machine_identity = match machine_identity.or(config.machine_identity) {
             Some(machine_identity) => machine_identity,
             None => MachineIdentity::from_metadata().await?,
         };
@@ -125,11 +125,14 @@ impl StaticConfig {
         Ok(config)
     }
 
-    pub async fn from_file(config_path: impl AsRef<Path>) -> Result<Self> {
+    pub async fn from_file(
+        config_path: impl AsRef<Path>,
+        machine_identity: Option<MachineIdentity>,
+    ) -> Result<Self> {
         let config_path = config_path.as_ref();
         let data = std::fs::read(config_path)
             .with_context(|| format!("unable to read config file: {}", config_path.display()))?;
-        Self::new(&data).await
+        Self::new(&data, machine_identity).await
     }
 
     pub fn from_env() -> Result<Self> {

src/agent/onefuzz-agent/src/main.rs

@@ -170,8 +170,6 @@ fn run(opt: RunOpt) -> Result<()> {
     if opt.redirect_output.is_some() {
         return redirect(opt);
     }
-    let opt_machine_id = opt.machine_id;
-    let opt_machine_name = opt.machine_name.clone();
     let rt = tokio::runtime::Runtime::new()?;
     let reset_lock = opt.reset_node_lock;
     let config = rt.block_on(load_config(opt));
@@ -184,15 +182,6 @@ fn run(opt: RunOpt) -> Result<()> {
 
     let config = config?;
 
-    let config = StaticConfig {
-        machine_identity: MachineIdentity {
-            machine_id: opt_machine_id.unwrap_or(config.machine_identity.machine_id),
-            machine_name: opt_machine_name.unwrap_or(config.machine_identity.machine_name),
-            ..config.machine_identity
-        },
-        ..config
-    };
-
     if reset_lock {
         done::remove_done_lock(config.machine_identity.machine_id)?;
     } else if done::is_agent_done(config.machine_identity.machine_id)? {
@@ -218,10 +207,18 @@ fn run(opt: RunOpt) -> Result<()> {
 }
 
 async fn load_config(opt: RunOpt) -> Result<StaticConfig> {
-    info!("loading supervisor agent config");
+    info!("loading supervisor agent config: {:?}", opt);
+    let opt_machine_id = opt.machine_id;
+    let opt_machine_name = opt.machine_name.clone();
+
+    let machine_identity = opt_machine_id.map(|machine_id| MachineIdentity {
+        machine_id,
+        machine_name: opt_machine_name.unwrap_or(format!("{}", machine_id)),
+        scaleset_name: None,
+    });
 
     let config = match &opt.config_path {
-        Some(config_path) => StaticConfig::from_file(config_path).await?,
+        Some(config_path) => StaticConfig::from_file(config_path, machine_identity).await?,
         None => StaticConfig::from_env()?,
     };
 

src/agent/onefuzz/src/auth.rs

@@ -113,9 +113,11 @@ impl ClientCredentials {
     pub async fn access_token(&self) -> Result<AccessToken> {
         let (authority, scope) = {
             let url = Url::parse(&self.resource.clone())?;
-            let host = url.host_str().ok_or_else(|| {
+            let port = url.port().map(|p| format!(":{}", p)).unwrap_or_default();
+            let host_name = url.host_str().ok_or_else(|| {
                 anyhow::format_err!("resource URL does not have a host string: {}", url)
             })?;
+            let host = format!("{}{}", host_name, port);
             if let Some(domain) = &self.multi_tenant_domain {
                 let instance: Vec<&str> = host.split('.').collect();
                 (