Merge branch 'main' into tevoinea/FuzzCoverageRecording

Author: tevoinea

.devcontainer/Dockerfile

@@ -5,7 +5,7 @@ ARG VARIANT="ubuntu-22.04"
 FROM mcr.microsoft.com/devcontainers/base:${VARIANT}
 
 # note: keep this in sync with .github/workflows/ci.yml
-ARG RUSTVERSION="1.71"
+ARG RUSTVERSION="1.71.1"
 
 # Install packages required for build:
 RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \

.devcontainer/devcontainer.json

@@ -13,6 +13,7 @@
 					"**/target/**": true
 				},
 				"lldb.executable": "/usr/bin/lldb",
+				"dotnet.server.useOmnisharp": true,
 				"omnisharp.enableEditorConfigSupport": true,
 				"omnisharp.enableRoslynAnalyzers": true,
 				"python.defaultInterpreterPath": "/workspaces/onefuzz/src/venv/bin/python",
@@ -48,4 +49,4 @@
 	"features": {
 		"ghcr.io/devcontainers/features/azure-cli:1": {}
 	}
-}
+}

.gitattributes

@@ -1,2 +1,3 @@
-* text=auto
-*.ps1 text=crlf
+*     text=auto
+*.ps1 text eol=crlf
+*.sh  text eol=lf

.github/workflows/ci.yml

@@ -16,7 +16,7 @@ concurrency:
 
 env:
   CARGO_TERM_COLOR: always
-  ACTIONS_CACHE_KEY_DATE: 2023-06-19
+  ACTIONS_CACHE_KEY_DATE: 2023-08-10
   CI: true
 
 jobs:
@@ -45,7 +45,7 @@ jobs:
       - name: Install specific Rust version
         uses: dtolnay/rust-toolchain@55c7845fad90d0ae8b2e83715cb900e5e861e8cb # pinned latest master as of 2022-10-08
         with:
-          toolchain: "1.71" # note: keep this in sync with .devcontainer/Dockerfile
+          toolchain: "1.71.1" # note: keep this in sync with .devcontainer/Dockerfile
           components: clippy, rustfmt, llvm-tools-preview
       - name: Setup Rust problem-matchers
         uses: r7kamura/rust-problem-matchers@d58b70c4a13c4866d96436315da451d8106f8f08 # pinned to 1.3.0
@@ -79,16 +79,24 @@ jobs:
           key: ${{env.ACTIONS_CACHE_KEY_DATE}} # additional key for cache-busting
           workspaces: src/agent
       - name: Linux Prereqs
-        if: runner.os == 'Linux' && steps.cache-agent-artifacts.outputs.cache-hit != 'true'
+        if: runner.os == 'Linux'
         run: |
           sudo apt-get -y update
-          sudo apt-get -y install libssl-dev libunwind-dev build-essential pkg-config
+          sudo apt-get -y install libssl-dev libunwind-dev build-essential pkg-config clang
+      - name: Clone onefuzz-samples
+        run: git clone https://github.com/microsoft/onefuzz-samples
+      - name: Prepare for agent integration tests
+        shell: bash
+        working-directory: ./onefuzz-samples/examples/simple-libfuzzer
+        run: |
+          make
+          mkdir -p ../../../src/agent/onefuzz-task/tests/targets/simple
+          cp fuzz.exe ../../../src/agent/onefuzz-task/tests/targets/simple/fuzz.exe
+          cp *.pdb ../../../src/agent/onefuzz-task/tests/targets/simple/ 2>/dev/null || :
       - name: Install Rust Prereqs
-        if: steps.rust-build-cache.outputs.cache-hit != 'true' && steps.cache-agent-artifacts.outputs.cache-hit != 'true'
         shell: bash
         run: src/ci/rust-prereqs.sh
       - run: src/ci/agent.sh
-        if: steps.cache-agent-artifacts.outputs.cache-hit != 'true'
         shell: bash
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
@@ -375,6 +383,10 @@ jobs:
         with:
           # use global.json to install the correct version
           global-json-file: global.json
+      - name: install llVM
+        run: |
+          choco install llvm --version 16.0.6
+        shell: powershell
       - run: src/ci/dotnet-fuzzing-tools.ps1
         shell: pwsh
       - uses: actions/upload-artifact@v3
@@ -527,7 +539,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: src/integration-tests/artifacts
-          key: integration-tests|linux|${{ hashFiles('src/integration-tests/**/*') }}
+          key: integration-tests|linux|${{ env.ACTIONS_CACHE_KEY_DATE }}|${{ hashFiles('src/integration-tests/**/*') }}
       - name: Build integration tests
         if: steps.cache-integration-tests.outputs.cache-hit != 'true'
         run: |
@@ -538,9 +550,11 @@ jobs:
 
           mkdir -p artifacts/linux-libfuzzer
           mkdir -p artifacts/linux-libfuzzer-with-options
+          mkdir -p artifacts/mariner-libfuzzer
           (cd libfuzzer ; make )
           cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/linux-libfuzzer
           cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/linux-libfuzzer-with-options
+          cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/mariner-libfuzzer
 
           mkdir -p artifacts/linux-libfuzzer-regression
           (cd libfuzzer-regression ; make )
@@ -590,15 +604,17 @@ jobs:
           name: artifact-integration-tests-linux
           path: src/integration-tests/artifacts
   build-integration-tests-windows:
-    runs-on: windows-2019
+    runs-on: windows-2022
     steps:
       - uses: actions/checkout@v3
       - name: Cache integration tests
         id: cache-integration-tests
         uses: actions/cache@v3
         with:
           path: src/integration-tests/artifacts
-          key: integration-tests|windows|${{ hashFiles('src/integration-tests/**/*') }}
+          key: integration-tests|windows|${{ env.ACTIONS_CACHE_KEY_DATE }}|${{ hashFiles('src/integration-tests/**/*') }}
+      - name: Setup C/C++ environment
+        uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # pinned to v1.12.1
       - name: Build integration tests
         if: steps.cache-integration-tests.outputs.cache-hit != 'true'
         run: |
@@ -609,33 +625,6 @@ jobs:
           choco install make
           $env:Path += ";C:\Program Files\LLVM\bin;C:\ProgramData\chocolatey\bin"
 
-          # WORKAROUND: effectively downgrade the default Windows 10 SDK version.
-          #
-          # This ensures we link against a version of the SDK which won't trigger a
-          # startup bug in the LLVM-shipped ASAN runtime.
-
-          # Assume a default MSVC 2019 install path.
-          $MsvcDir = 'C:/Program Files (x86)/Microsoft Visual Studio/2019/Enterprise/VC/Tools/MSVC'
-
-          # Assume that `$MsvcDir` only contains version-named subdirectories.
-          $MsvcVersion = ((Get-ChildItem $MsvcDir).name | Sort-Object -Descending)[0]
-          $MsvcLib = "${MsvcDir}/${MsvcVersion}/lib/x64"
-
-          # Known "good" (non-bug-surfacing) version.
-          $WindowsSdkVersion = '10.0.18362.0'
-
-          # Assume default install path.
-          $WindowsSdkDir = 'C:/Program Files (x86)/Windows Kits/10'
-          $WindowsSdkLib = "${WindowsSdkDir}/Lib/${WindowsSdkVersion}"
-          $WindowsSdkInclude = "${WindowsSdkDir}/Include/${WindowsSdkVersion}"
-
-          # Used by `clang.exe`.
-          $env:CPATH = $WindowsSdkInclude
-          $env:LIBRARY_PATH = "${MsvcLib};${WindowsSdkLib}/ucrt/x64;${WindowsSdkLib}/um/x64"
-
-          # Used by `link.exe`.
-          $env:LIB = $env:LIBRARY_PATH
-
           cd src/integration-tests
 
           mkdir artifacts/windows-libfuzzer

CHANGELOG.md

@@ -7,6 +7,105 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 8.8.0
+
+### Added
+
+* Agent: Added Mariner Linux support for agent VMs [#3306](https://github.com/microsoft/onefuzz/pull/3306)
+* Service: Added support for custom ado fields that mark work items as duplicate [#3467](https://github.com/microsoft/onefuzz/pull/3467)
+* Service: Permanently store OneFuzz job result data - # crashing input, # regression crashing input, etc. - in Azure storage [#3380](https://github.com/microsoft/onefuzz/pull/3380), [#3439](https://github.com/microsoft/onefuzz/pull/3439)
+* Service: Added validation for Iteration/AreaPath on notifications when a job is submitted with a notification config and for `onefuzz debug notification test_template` [#3386](https://github.com/microsoft/onefuzz/pull/3386)
+
+### Changed
+
+* Agent: Updated libfuzzer-fuzz basic template to include required args and make it match cli [#3429](https://github.com/microsoft/onefuzz/pull/3429)
+* Agent: Downgraded some debug logs from warn to debug [#3450](https://github.com/microsoft/onefuzz/pull/3450)
+* CLI: Removed CLI commands from the local fuzzing tasks as they can now be described via yaml template [#3428](https://github.com/microsoft/onefuzz/pull/3428)
+* Service: AutoScale table entries are now deleted on VMSS shutdown [#3455](https://github.com/microsoft/onefuzz/pull/3455)
+
+### Fixed
+
+* Agent: Fixed local path generation [#3432](https://github.com/microsoft/onefuzz/pull/3432), [#3460](https://github.com/microsoft/onefuzz/pull/3460)
+
+## 8.7.1
+
+### Fixed
+
+* Service: Removed deprecated Azure retention policy setting that was causing scaleset deployment errors [#3452](https://github.com/microsoft/onefuzz/pull/3452)
+
+## 8.7.0
+
+### Added
+
+* Agent: Added a snapshot-based test to coverage implementation [#3368](https://github.com/microsoft/onefuzz/pull/3368)
+* Agent/CLI/Service: Added ability to capture crash dumps from libfuzzer, when provided [#2793](https://github.com/microsoft/onefuzz/pull/2793) [#3409](https://github.com/microsoft/onefuzz/pull/3409)
+* CLI/Service: Implemented `--with_tasks ` option for `onefuzz jobs get` command to expand the task information [#3343](https://github.com/microsoft/onefuzz/pull/3343)
+
+### Changed
+
+* Agent: Migrated all the task types to the template model [#3397](https://github.com/microsoft/onefuzz/pull/3307)
+* Agent: Removed `srcview` code from OneFuzz since it is not currently utilized [#3376](https://github.com/microsoft/onefuzz/pull/3376)
+* Agent: Updated default windows VM image to windows 11 [#3374](https://github.com/microsoft/onefuzz/pull/3374)
+* Agent: Migrated `winapi` to `windows-rs`, the newer Microsoft supported version of the Windows API bindings for Rust [#3050](https://github.com/microsoft/onefuzz/pull/3050)
+* Deployment: Updated the default deployment option for `EnableWorkItemCreation` feature flag to be enabled [#3387](https://github.com/microsoft/onefuzz/pull/3387)
+
+### Fixed
+
+* Agent: Deserialize the coverage files directly into the output files [#3410](https://github.com/microsoft/onefuzz/pull/3410)
+* Agent/Deployment/Service: Bumped several C#, Python, and Rust dependencies as well as the Rust edition across all Rust crates [#3396](https://github.com/microsoft/onefuzz/pull/3396), [#3161](https://github.com/microsoft/onefuzz/pull/3161), [#3346](https://github.com/microsoft/onefuzz/pull/3346), [#3391](https://github.com/microsoft/onefuzz/pull/3391), [#2870](https://github.com/microsoft/onefuzz/pull/2870), [#3392](https://github.com/microsoft/onefuzz/pull/3392), [#3402](https://github.com/microsoft/onefuzz/pull/3402)
+* Agent: Fixed a bug in agent `DirectoryMonitor` by adding error tolerance when attempting to fetch metadata for `CreateKind::Any` or `CreateKind::Other` events [#3393](https://github.com/microsoft/onefuzz/pull/3393)
+* Service: Fixed tag shadowing in logging by giving precedence to the tags produced by log messages over the tags added prior to the call, when the tag names clashed [#3388](https://github.com/microsoft/onefuzz/pull/3388)
+
+## 8.6.3
+
+### Fixed
+
+* Service: Fixed another duplicate Azure DevOps work item creation case by handling `Microsoft.VSTS.Common.ResolvedReason` field when present [#3383](https://github.com/microsoft/onefuzz/pull/3383)
+
+## 8.6.2
+
+### Fixed
+
+* Agent: Fixed tasks hanging when shutting down by forcefully shutting down the runtime before exiting the main task [#3378](https://github.com/microsoft/onefuzz/pull/3378)
+* Service: Refactored Azure DevOps template rendering to fix duplicate bugs being filed due to title truncation and added several validation tests in this area [#3370](https://github.com/microsoft/onefuzz/pull/3370)
+
+## 8.6.1
+
+### Added
+
+* Service: Added feature flag to toggle Azure DevOps work item processing [#3353](https://github.com/microsoft/onefuzz/pull/3353)
+* Service: Requeue Azure DevOps notifications when the feature flag for work item processing is set to 'disabled' [#3358](https://github.com/microsoft/onefuzz/pull/3358)
+
+## 8.6.0
+
+### Added
+
+* Agent: Implemented `debuginfo` caching [#3280](https://github.com/microsoft/onefuzz/pull/3280)
+
+### Changed
+
+* Agent: Limit azcopy copy buffer to 512MB of RAM as the default maximum [#3293](https://github.com/microsoft/onefuzz/pull/3293)
+* Agent: Define local fuzzing tasks relationships through new templating model [#3117](https://github.com/microsoft/onefuzz/pull/3117)
+* Deployment: Replaced `--upgrade` flag with `--skip_aad_setup` flag in the deploy.py setup script [#3345](https://github.com/microsoft/onefuzz/pull/3345)
+* Service: Make `ServiceConfiguration` eagerly evaluated [#3136](https://github.com/microsoft/onefuzz/pull/3136)
+* Service: Improved `TimerRetention` performance through several UPN changes & fixes [#3289](https://github.com/microsoft/onefuzz/pull/3289)
+
+### Fixed
+
+* Agent: Fixed resolution of sibling .NET DLLs [#3325](https://github.com/microsoft/onefuzz/pull/3325)
+* Agent/Service: Bumped several C# and Rust dependencies [#3319](https://github.com/microsoft/onefuzz/pull/3319), [#3320](https://github.com/microsoft/onefuzz/pull/3320), [#3317](https://github.com/microsoft/onefuzz/pull/3317), [#3297](https://github.com/microsoft/onefuzz/pull/3297), [#3301](https://github.com/microsoft/onefuzz/pull/3301), [#3291](https://github.com/microsoft/onefuzz/pull/3291), [#3195](https://github.com/microsoft/onefuzz/pull/3195), [#3328](https://github.com/microsoft/onefuzz/pull/3328)
+* CLI: Look for azcopy.exe in environment variable `AZCOPY` and determine if it's actually referencing a directory [#3344](https://github.com/microsoft/onefuzz/pull/3344)
+* CLI: Updated `repro get_files` to handle regression reports [#3340](https://github.com/microsoft/onefuzz/pull/3340)
+* CLI: Fixed missing `target_timeout` setting in the Libfuzzer basic template [#3334](https://github.com/microsoft/onefuzz/pull/3334)
+* CLI: Fixed false 'missing' dependency warning [#3331](https://github.com/microsoft/onefuzz/pull/3331)
+* CLI: Fixed the `debug notification test_template` command expecting a `task_id`  [#3308](https://github.com/microsoft/onefuzz/pull/3308)
+* Deployment: Update App Registration redirect URIs if deployment uses a custom domain [#3341](https://github.com/microsoft/onefuzz/pull/3341)
+* Service: Fixed links in bugs filed from regression reports by populating `InputBlob` when possible [#3342](https://github.com/microsoft/onefuzz/pull/3342)
+* Service: Fixed several storage issues to improve platform performance and reduce spurious `404`s [#3313](https://github.com/microsoft/onefuzz/pull/3313)
+* Service: Added extra logging when `System.Title` is too long [#3332](https://github.com/microsoft/onefuzz/pull/3332)
+* Service: Render `System.Title` before trying to trim it to the max allowed size [#3329](https://github.com/microsoft/onefuzz/pull/3329)
+* Service: Differentiate `INVALID_JOB` and `INVALID_TASK` error codes [#3318](https://github.com/microsoft/onefuzz/pull/3318)
+
 ## 8.5.0
 
 ### Added

CURRENT_VERSION

@@ -1 +1 @@
-8.5.0
+8.8.0

README.md

@@ -1,5 +1,20 @@
 # <img src="docs/onefuzz_text.svg" height="120" alt="OneFuzz" />
 
+# IMPORTANT NOTICE
+
+**_Since September 2020 when OneFuzz was first open sourced, we’ve been on a journey to create a best-in-class orchestrator for running fuzzers, driving security and quality into our products._**
+ 
+ 
+**_Initially launched by a small group in MSR, OneFuzz has now become a significant internal platform within Microsoft. As such, we are regretfully archiving the project to focus our attention on becoming a more deeply integrated service within the company. Unfortunately, we aren’t a large enough team to live in both the open-source world and the internal Microsoft world with its own unique set of requirements._**
+ 
+**_Our current plan is to archive the project in the next few months. That means we’ll still be making updates for a little while. Of course, even after it’s archived, you’ll still be able to fork it and make the changes you need. Once we’ve decided on a specific date for archiving, we’ll update this readme._**
+ 
+**_Thanks for taking the journey with us._**
+
+**_The OneFuzz team._**
+
+---
+
 [![Onefuzz build status](https://github.com/microsoft/onefuzz/workflows/Build/badge.svg?branch=main)](https://github.com/microsoft/onefuzz/actions/workflows/ci.yml?query=branch%3Amain)
 
 ## A self-hosted Fuzzing-As-A-Service platform