Skip to content

Commit ec89018

Browse files
Porgeschkeita
Porges
authored and
chkeita
committed
Support for retention policies on containers (microsoft#3501)
- [x] ability to specify a retention period on a container, which applies to newly-created blobs - [x] specify default retention periods in templates from CLI side There's a small breaking change to the Python JobHelper class.
1 parent b0ce6e7 commit ec89018

File tree

19 files changed

+393
-184
lines changed

19 files changed

+393
-184
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ jobs:
123123
- uses: actions/checkout@v3
124124
- uses: actions/setup-python@v4
125125
with:
126-
python-version: 3.7
126+
python-version: "3.10"
127127
- name: lint
128128
shell: bash
129129
run: src/ci/check-check-pr.sh
@@ -137,7 +137,7 @@ jobs:
137137
shell: bash
138138
- uses: actions/setup-python@v4
139139
with:
140-
python-version: 3.7
140+
python-version: "3.10"
141141
- uses: actions/download-artifact@v3
142142
with:
143143
name: artifact-onefuzztypes
@@ -190,7 +190,7 @@ jobs:
190190
- uses: actions/checkout@v3
191191
- uses: actions/setup-python@v4
192192
with:
193-
python-version: 3.8
193+
python-version: "3.10"
194194
- name: lint
195195
shell: bash
196196
run: |
@@ -208,7 +208,7 @@ jobs:
208208
- uses: actions/checkout@v3
209209
- uses: actions/setup-python@v4
210210
with:
211-
python-version: 3.8
211+
python-version: "3.10"
212212
- name: lint
213213
shell: bash
214214
run: |
@@ -224,7 +224,7 @@ jobs:
224224
- run: src/ci/set-versions.sh
225225
- uses: actions/setup-python@v4
226226
with:
227-
python-version: 3.7
227+
python-version: "3.10"
228228
- run: src/ci/onefuzztypes.sh
229229
- uses: actions/upload-artifact@v3
230230
with:
@@ -481,7 +481,7 @@ jobs:
481481
path: artifacts
482482
- uses: actions/setup-python@v4
483483
with:
484-
python-version: 3.7
484+
python-version: "3.10"
485485
- name: Lint
486486
shell: bash
487487
run: |

‎src/ApiService/ApiService/FeatureFlags.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,5 @@ public static class FeatureFlagConstants {
88
public const string EnableBlobRetentionPolicy = "EnableBlobRetentionPolicy";
99
public const string EnableDryRunBlobRetention = "EnableDryRunBlobRetention";
1010
public const string EnableWorkItemCreation = "EnableWorkItemCreation";
11+
public const string EnableContainerRetentionPolicies = "EnableContainerRetentionPolicies";
1112
}

‎src/ApiService/ApiService/Functions/QueueFileChanges.cs‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System.Text.Json;
22
using System.Text.Json.Nodes;
3+
using System.Threading.Tasks;
34
using Azure.Core;
45
using Microsoft.Azure.Functions.Worker;
56
using Microsoft.Extensions.Logging;
@@ -54,14 +55,16 @@ public async Async.Task Run(
5455
return;
5556
}
5657

58+
var storageAccount = new ResourceIdentifier(topicElement.GetString()!);
59+
5760
try {
5861
// Setting isLastRetryAttempt to false will rethrow any exceptions
5962
// With the intention that the azure functions runtime will handle requeing
6063
// the message for us. The difference is for the poison queue, we're handling the
6164
// requeuing ourselves because azure functions doesn't support retry policies
6265
// for queue based functions.
6366

64-
var result = await FileAdded(fileChangeEvent, isLastRetryAttempt: false);
67+
var result = await FileAdded(storageAccount, fileChangeEvent, isLastRetryAttempt: false);
6568
if (!result.IsOk && result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED) {
6669
await RequeueMessage(msg, TimeSpan.FromDays(1));
6770
}
@@ -71,16 +74,47 @@ public async Async.Task Run(
7174
}
7275
}
7376

74-
private async Async.Task<OneFuzzResultVoid> FileAdded(JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
77+
private async Async.Task<OneFuzzResultVoid> FileAdded(ResourceIdentifier storageAccount, JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
7578
var data = fileChangeEvent.RootElement.GetProperty("data");
7679
var url = data.GetProperty("url").GetString()!;
7780
var parts = url.Split("/").Skip(3).ToList();
7881

79-
var container = parts[0];
82+
var container = Container.Parse(parts[0]);
8083
var path = string.Join('/', parts.Skip(1));
8184

82-
_log.LogInformation("file added : {Container} - {Path}", container, path);
83-
return await _notificationOperations.NewFiles(Container.Parse(container), path, isLastRetryAttempt);
85+
_log.LogInformation("file added : {Container} - {Path}", container.String, path);
86+
87+
var (_, result) = await (
88+
ApplyRetentionPolicy(storageAccount, container, path),
89+
_notificationOperations.NewFiles(container, path, isLastRetryAttempt));
90+
91+
return result;
92+
}
93+
94+
private async Async.Task<bool> ApplyRetentionPolicy(ResourceIdentifier storageAccount, Container container, string path) {
95+
if (await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableContainerRetentionPolicies)) {
96+
// default retention period can be applied to the container
97+
// if one exists, we will set the expiry date on the newly-created blob, if it doesn't already have one
98+
var account = await _storage.GetBlobServiceClientForAccount(storageAccount);
99+
var containerClient = account.GetBlobContainerClient(container.String);
100+
var containerProps = await containerClient.GetPropertiesAsync();
101+
var retentionPeriod = RetentionPolicyUtils.GetContainerRetentionPeriodFromMetadata(containerProps.Value.Metadata);
102+
if (!retentionPeriod.IsOk) {
103+
_log.LogError("invalid retention period: {Error}", retentionPeriod.ErrorV);
104+
} else if (retentionPeriod.OkV is TimeSpan period) {
105+
var blobClient = containerClient.GetBlobClient(path);
106+
var tags = (await blobClient.GetTagsAsync()).Value.Tags;
107+
var expiryDate = DateTime.UtcNow + period;
108+
var tag = RetentionPolicyUtils.CreateExpiryDateTag(DateOnly.FromDateTime(expiryDate));
109+
if (tags.TryAdd(tag.Key, tag.Value)) {
110+
_ = await blobClient.SetTagsAsync(tags);
111+
_log.LogInformation("applied container retention policy ({Policy}) to {Path}", period, path);
112+
return true;
113+
}
114+
}
115+
}
116+
117+
return false;
84118
}
85119

86120
private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null) {

‎src/ApiService/ApiService/OneFuzzTypes/Enums.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ public enum ErrorCode {
5050
ADO_WORKITEM_PROCESSING_DISABLED = 494,
5151
ADO_VALIDATION_INVALID_PATH = 495,
5252
ADO_VALIDATION_INVALID_PROJECT = 496,
53+
INVALID_RETENTION_PERIOD = 497,
5354
// NB: if you update this enum, also update enums.py
5455
}
5556

‎src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,12 @@ public NotificationOperations(ILogger<NotificationOperations> log, IOnefuzzConte
2222

2323
}
2424
public async Async.Task<OneFuzzResultVoid> NewFiles(Container container, string filename, bool isLastRetryAttempt) {
25-
var result = OneFuzzResultVoid.Ok;
26-
2725
// We don't want to store file added events for the events container because that causes an infinite loop
2826
if (container == WellKnownContainers.Events) {
29-
return result;
27+
return Result.Ok();
3028
}
3129

30+
var result = OneFuzzResultVoid.Ok;
3231
var notifications = GetNotifications(container);
3332
var hasNotifications = await notifications.AnyAsync();
3433
var reportOrRegression = await _context.Reports.GetReportOrRegression(container, filename, expectReports: hasNotifications);

‎src/ApiService/ApiService/onefuzzlib/RententionPolicy.cs‎

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎src/ApiService/ApiService/onefuzzlib/RetentionPolicy.cs‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
using System.Xml;
2+
3+
namespace Microsoft.OneFuzz.Service;
4+
5+
6+
public interface IRetentionPolicy {
7+
DateOnly GetExpiryDate();
8+
}
9+
10+
public class RetentionPolicyUtils {
11+
public const string EXPIRY_TAG = "Expiry";
12+
public static KeyValuePair<string, string> CreateExpiryDateTag(DateOnly expiryDate) =>
13+
new(EXPIRY_TAG, expiryDate.ToString());
14+
15+
public static DateOnly? GetExpiryDateTagFromTags(IDictionary<string, string>? blobTags) {
16+
if (blobTags != null &&
17+
blobTags.TryGetValue(EXPIRY_TAG, out var expiryTag) &&
18+
!string.IsNullOrWhiteSpace(expiryTag) &&
19+
DateOnly.TryParse(expiryTag, out var expiryDate)) {
20+
return expiryDate;
21+
}
22+
return null;
23+
}
24+
25+
public static string CreateExpiredBlobTagFilter() => $@"""{EXPIRY_TAG}"" <= '{DateOnly.FromDateTime(DateTime.UtcNow)}'";
26+
27+
// NB: this must match the value used on the CLI side
28+
public const string CONTAINER_RETENTION_KEY = "onefuzz_retentionperiod";
29+
30+
public static OneFuzzResult<TimeSpan?> GetContainerRetentionPeriodFromMetadata(IDictionary<string, string>? containerMetadata) {
31+
if (containerMetadata is not null &&
32+
containerMetadata.TryGetValue(CONTAINER_RETENTION_KEY, out var retentionString) &&
33+
!string.IsNullOrWhiteSpace(retentionString)) {
34+
try {
35+
return Result.Ok<TimeSpan?>(XmlConvert.ToTimeSpan(retentionString));
36+
} catch (Exception ex) {
37+
return Error.Create(ErrorCode.INVALID_RETENTION_PERIOD, ex.Message);
38+
}
39+
}
40+
41+
return Result.Ok<TimeSpan?>(null);
42+
}
43+
}

‎src/cli/examples/domato.py‎

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ def upload_to_fuzzer_container(of: Onefuzz, fuzzer_name: str, fuzzer_url: str) -
6767

6868

6969
def upload_to_setup_container(of: Onefuzz, helper: JobHelper, setup_dir: str) -> None:
70-
setup_sas = of.containers.get(helper.containers[ContainerType.setup]).sas_url
70+
setup_sas = of.containers.get(helper.container_name(ContainerType.setup)).sas_url
7171
if AZCOPY_PATH is None:
7272
raise Exception("missing azcopy")
7373
command = [AZCOPY_PATH, "sync", setup_dir, setup_sas]
@@ -143,13 +143,16 @@ def main() -> None:
143143
helper.create_containers()
144144
helper.setup_notifications(notification_config)
145145
upload_to_setup_container(of, helper, args.setup_dir)
146-
add_setup_script(of, helper.containers[ContainerType.setup])
146+
add_setup_script(of, helper.container_name(ContainerType.setup))
147147

148148
containers = [
149-
(ContainerType.setup, helper.containers[ContainerType.setup]),
150-
(ContainerType.crashes, helper.containers[ContainerType.crashes]),
151-
(ContainerType.reports, helper.containers[ContainerType.reports]),
152-
(ContainerType.unique_reports, helper.containers[ContainerType.unique_reports]),
149+
(ContainerType.setup, helper.container_name(ContainerType.setup)),
150+
(ContainerType.crashes, helper.container_name(ContainerType.crashes)),
151+
(ContainerType.reports, helper.container_name(ContainerType.reports)),
152+
(
153+
ContainerType.unique_reports,
154+
helper.container_name(ContainerType.unique_reports),
155+
),
153156
]
154157

155158
of.logger.info("Creating generic_crash_report task")
@@ -164,11 +167,11 @@ def main() -> None:
164167

165168
containers = [
166169
(ContainerType.tools, Container(FUZZER_NAME)),
167-
(ContainerType.setup, helper.containers[ContainerType.setup]),
168-
(ContainerType.crashes, helper.containers[ContainerType.crashes]),
170+
(ContainerType.setup, helper.container_name(ContainerType.setup)),
171+
(ContainerType.crashes, helper.container_name(ContainerType.crashes)),
169172
(
170173
ContainerType.readonly_inputs,
171-
helper.containers[ContainerType.readonly_inputs],
174+
helper.container_name(ContainerType.readonly_inputs),
172175
),
173176
]
174177

‎src/cli/examples/honggfuzz.py‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -88,13 +88,16 @@ def main() -> None:
8888
if args.inputs:
8989
helper.upload_inputs(args.inputs)
9090

91-
add_setup_script(of, helper.containers[ContainerType.setup])
91+
add_setup_script(of, helper.container_name(ContainerType.setup))
9292

9393
containers = [
94-
(ContainerType.setup, helper.containers[ContainerType.setup]),
95-
(ContainerType.crashes, helper.containers[ContainerType.crashes]),
96-
(ContainerType.reports, helper.containers[ContainerType.reports]),
97-
(ContainerType.unique_reports, helper.containers[ContainerType.unique_reports]),
94+
(ContainerType.setup, helper.container_name(ContainerType.setup)),
95+
(ContainerType.crashes, helper.container_name(ContainerType.crashes)),
96+
(ContainerType.reports, helper.container_name(ContainerType.reports)),
97+
(
98+
ContainerType.unique_reports,
99+
helper.container_name(ContainerType.unique_reports),
100+
),
98101
]
99102

100103
of.logger.info("Creating generic_crash_report task")
@@ -109,11 +112,11 @@ def main() -> None:
109112

110113
containers = [
111114
(ContainerType.tools, Container("honggfuzz")),
112-
(ContainerType.setup, helper.containers[ContainerType.setup]),
113-
(ContainerType.crashes, helper.containers[ContainerType.crashes]),
115+
(ContainerType.setup, helper.container_name(ContainerType.setup)),
116+
(ContainerType.crashes, helper.container_name(ContainerType.crashes)),
114117
(
115118
ContainerType.inputs,
116-
helper.containers[ContainerType.inputs],
119+
helper.container_name(ContainerType.inputs),
117120
),
118121
]
119122

‎src/cli/examples/llvm-source-coverage/source-coverage-libfuzzer.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,15 +74,15 @@ def main() -> None:
7474
helper.create_containers()
7575

7676
of.containers.files.upload_file(
77-
helper.containers[ContainerType.tools], f"{args.tools}/source-coverage.sh"
77+
helper.container_name(ContainerType.tools), f"{args.tools}/source-coverage.sh"
7878
)
7979

8080
containers = [
81-
(ContainerType.setup, helper.containers[ContainerType.setup]),
82-
(ContainerType.analysis, helper.containers[ContainerType.analysis]),
83-
(ContainerType.tools, helper.containers[ContainerType.tools]),
81+
(ContainerType.setup, helper.container_name(ContainerType.setup)),
82+
(ContainerType.analysis, helper.container_name(ContainerType.analysis)),
83+
(ContainerType.tools, helper.container_name(ContainerType.tools)),
8484
# note, analysis is typically for crashes, but this is analyzing inputs
85-
(ContainerType.crashes, helper.containers[ContainerType.inputs]),
85+
(ContainerType.crashes, helper.container_name(ContainerType.inputs)),
8686
]
8787

8888
of.logger.info("Creating generic_analysis task")

0 commit comments

Comments
 (0)